Merge c61e05e4f4 into a9e0fac9dc

src/ARMeilleure/Translation/PTC/Ptc.cs

@@ -3,6 +3,7 @@ using ARMeilleure.CodeGen.Linking;
 using ARMeilleure.CodeGen.Unwinding;
 using ARMeilleure.Common;
 using ARMeilleure.Memory;
+using ARMeilleure.State;
 using Ryujinx.Common;
 using Ryujinx.Common.Configuration;
 using Ryujinx.Common.Logging;
@@ -30,7 +31,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 6997; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 7007; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
@@ -185,6 +186,36 @@ namespace ARMeilleure.Translation.PTC
             InitializeCarriers();
         }
 
+        private bool ContainsBlacklistedFunctions()
+        {
+            List<ulong> blacklist = Profiler.GetBlacklistedFunctions();
+            bool containsBlacklistedFunctions = false;
+            _infosStream.Seek(0L, SeekOrigin.Begin);
+            bool foundBadFunction = false;
+
+            for (int index = 0; index < GetEntriesCount(); index++)
+            {
+                InfoEntry infoEntry = DeserializeStructure<InfoEntry>(_infosStream);
+                foreach (ulong address in blacklist)
+                {
+                    if (infoEntry.Address == address)
+                    {
+                        containsBlacklistedFunctions = true;
+                        Logger.Warning?.Print(LogClass.Ptc, "PPTC cache invalidated: Found blacklisted functions in PPTC cache");
+                        foundBadFunction = true;
+                        break;
+                    }
+                }
+
+                if (foundBadFunction)
+                {
+                    break;
+                }
+            }
+
+            return containsBlacklistedFunctions;
+        }
+
         private void PreLoad()
         {
             string fileNameActual = $"{CachePathActual}.cache";
@@ -533,7 +564,7 @@ namespace ARMeilleure.Translation.PTC
 
         public void LoadTranslations(Translator translator)
         {
-            if (AreCarriersEmpty())
+            if (AreCarriersEmpty() || ContainsBlacklistedFunctions())
             {
                 return;
             }
@@ -836,10 +867,18 @@ namespace ARMeilleure.Translation.PTC
                 while (profiledFuncsToTranslate.TryDequeue(out var item))
                 {
                     ulong address = item.address;
+                    ExecutionMode executionMode = item.funcProfile.Mode;
+                    bool highCq = item.funcProfile.HighCq;
 
                     Debug.Assert(Profiler.IsAddressInStaticCodeRange(address));
 
-                    TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
+                    TranslatedFunction func = translator.Translate(address, executionMode, highCq);
+
+                    if (func == null)
+                    {
+                        Profiler.UpdateEntry(address, executionMode, true, true);
+                        continue;
+                    }
 
                     bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func);
 
@@ -886,7 +925,14 @@ namespace ARMeilleure.Translation.PTC
 
             PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);
 
+            if (_translateCount == _translateTotalCount)
+            {
                 Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
+            }
+            else
+            {
+                Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | {_translateTotalCount - _translateCount} function{(_translateTotalCount - _translateCount != 1 ? "s" : "")} blacklisted | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
+            }
 
             Thread preSaveThread = new(PreSave)
             {

src/ARMeilleure/Translation/PTC/PtcProfiler.cs

@@ -23,10 +23,11 @@ namespace ARMeilleure.Translation.PTC
     {
         private const string OuterHeaderMagicString = "Pohd\0\0\0\0";
 
-        private const uint InternalVersion = 5518; //! Not to be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 7007; //! Not to be incremented manually for each change to the ARMeilleure project.
 
         private static readonly uint[] _migrateInternalVersions = {
             1866,
+            5518,
         };
 
         private const int SaveInterval = 30; // Seconds.
@@ -74,20 +75,30 @@ namespace ARMeilleure.Translation.PTC
             Enabled = false;
         }
 
-        public void AddEntry(ulong address, ExecutionMode mode, bool highCq)
+        public void AddEntry(ulong address, ExecutionMode mode, bool highCq, bool blacklist = false)
         {
             if (IsAddressInStaticCodeRange(address))
             {
                 Debug.Assert(!highCq);
 
+                if (blacklist)
+                {
                     lock (_lock)
                     {
-                    ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false));
+                        ProfiledFuncs[address] = new FuncProfile(mode, highCq: false, true);
+                    }
+                }
+                else
+                {
+                    lock (_lock)
+                    {
+                        ProfiledFuncs.TryAdd(address, new FuncProfile(mode, highCq: false, false));
+                    }
                 }
             }
         }
 
-        public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq)
+        public void UpdateEntry(ulong address, ExecutionMode mode, bool highCq, bool? blacklist = null)
         {
             if (IsAddressInStaticCodeRange(address))
             {
@@ -97,7 +108,7 @@ namespace ARMeilleure.Translation.PTC
                 {
                     Debug.Assert(ProfiledFuncs.ContainsKey(address));
 
-                    ProfiledFuncs[address] = new FuncProfile(mode, highCq: true);
+                    ProfiledFuncs[address] = new FuncProfile(mode, highCq: true, blacklist ?? ProfiledFuncs[address].Blacklist);
                 }
             }
         }
@@ -113,7 +124,7 @@ namespace ARMeilleure.Translation.PTC
 
             foreach (var profiledFunc in ProfiledFuncs)
             {
-                if (!funcs.ContainsKey(profiledFunc.Key))
+                if (!funcs.ContainsKey(profiledFunc.Key) && !profiledFunc.Value.Blacklist)
                 {
                     profiledFuncsToTranslate.Enqueue((profiledFunc.Key, profiledFunc.Value));
                 }
@@ -128,6 +139,24 @@ namespace ARMeilleure.Translation.PTC
             ProfiledFuncs.TrimExcess();
         }
 
+        public List<ulong> GetBlacklistedFunctions()
+        {
+            List<ulong> funcs = new List<ulong>();
+
+            foreach (var profiledFunc in ProfiledFuncs)
+            {
+                if (profiledFunc.Value.Blacklist)
+                {
+                    if (!funcs.Contains(profiledFunc.Key))
+                    {
+                        funcs.Add(profiledFunc.Key);
+                    }
+                }
+            }
+
+            return funcs;
+        }
+
         public void PreLoad()
         {
             _lastHash = default;
@@ -218,13 +247,18 @@ namespace ARMeilleure.Translation.PTC
                     return false;
                 }
 
+                Func<ulong, FuncProfile, (ulong, FuncProfile)> migrateEntryFunc = null;
+
                 switch (outerHeader.InfoFileVersion)
                 {
                     case InternalVersion:
                         ProfiledFuncs = Deserialize(stream);
                         break;
                     case 1866:
-                        ProfiledFuncs = Deserialize(stream, (address, profile) => (address + 0x500000UL, profile));
+                        migrateEntryFunc = (address, profile) => (address + 0x500000UL, profile);
+                        goto case 5518;
+                    case 5518:
+                        ProfiledFuncs = DeserializeAddBlacklist(stream, migrateEntryFunc);
                         break;
                     default:
                         Logger.Error?.Print(LogClass.Ptc, $"No migration path for {nameof(outerHeader.InfoFileVersion)} '{outerHeader.InfoFileVersion}'. Discarding cache.");
@@ -254,6 +288,16 @@ namespace ARMeilleure.Translation.PTC
             return DeserializeDictionary<ulong, FuncProfile>(stream, DeserializeStructure<FuncProfile>);
         }
 
+        private static Dictionary<ulong, FuncProfile> DeserializeAddBlacklist(Stream stream, Func<ulong, FuncProfile, (ulong, FuncProfile)> migrateEntryFunc = null)
+        {
+            if (migrateEntryFunc != null)
+            {
+                return DeserializeAndUpdateDictionary(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure<FuncProfilePreBlacklist>(stream)); }, migrateEntryFunc);
+            }
+
+            return DeserializeDictionary<ulong, FuncProfile>(stream, (Stream stream) => { return new FuncProfile(DeserializeStructure<FuncProfilePreBlacklist>(stream)); });
+        }
+
         private static ReadOnlySpan<byte> GetReadOnlySpan(MemoryStream memoryStream)
         {
             return new(memoryStream.GetBuffer(), (int)memoryStream.Position, (int)memoryStream.Length - (int)memoryStream.Position);
@@ -385,13 +429,35 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)]
+        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 6*/)]
         public struct FuncProfile
         {
             public ExecutionMode Mode;
             public bool HighCq;
+            public bool Blacklist;
 
-            public FuncProfile(ExecutionMode mode, bool highCq)
+            public FuncProfile(ExecutionMode mode, bool highCq, bool blacklist)
+            {
+                Mode = mode;
+                HighCq = highCq;
+                Blacklist = blacklist;
+            }
+
+            public FuncProfile(FuncProfilePreBlacklist fp)
+            {
+                Mode = fp.Mode;
+                HighCq = fp.HighCq;
+                Blacklist = false;
+            }
+        }
+
+        [StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 5*/)]
+        public struct FuncProfilePreBlacklist
+        {
+            public ExecutionMode Mode;
+            public bool HighCq;
+
+            public FuncProfilePreBlacklist(ExecutionMode mode, bool highCq)
             {
                 Mode = mode;
                 HighCq = highCq;

src/ARMeilleure/Translation/Translator.cs

@@ -249,6 +249,11 @@ namespace ARMeilleure.Translation
 
             ControlFlowGraph cfg = EmitAndGetCFG(context, blocks, out Range funcRange, out Counter<uint> counter);
 
+            if (cfg == null)
+            {
+                return null;
+            }
+
             ulong funcSize = funcRange.End - funcRange.Start;
 
             Logger.EndPass(PassName.Translation, cfg);
@@ -407,6 +412,11 @@ namespace ARMeilleure.Translation
                         if (opCode.Instruction.Emitter != null)
                         {
                             opCode.Instruction.Emitter(context);
+                            if (opCode.Instruction.Name == InstName.Und && blkIndex == 0)
+                            {
+                                range = new Range(rangeStart, rangeEnd);
+                                return null;
+                            }
                         }
                         else
                         {

src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs

@@ -20,6 +20,7 @@ namespace Ryujinx.HLE.HOS
         private readonly string _titleIdText;
         private readonly string _displayVersion;
         private readonly bool _diskCacheEnabled;
+        private readonly string _diskCacheSelector;
         private readonly ulong _codeAddress;
         private readonly ulong _codeSize;
 
@@ -31,6 +32,7 @@ namespace Ryujinx.HLE.HOS
             string titleIdText,
             string displayVersion,
             bool diskCacheEnabled,
+            string diskCacheSelector,
             ulong codeAddress,
             ulong codeSize)
         {
@@ -39,6 +41,7 @@ namespace Ryujinx.HLE.HOS
             _titleIdText = titleIdText;
             _displayVersion = displayVersion;
             _diskCacheEnabled = diskCacheEnabled;
+            _diskCacheSelector = diskCacheSelector;
             _codeAddress = codeAddress;
             _codeSize = codeSize;
         }
@@ -114,7 +117,7 @@ namespace Ryujinx.HLE.HOS
                 }
             }
 
-            DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, "default"); //Ready for exefs profiles
+            DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize, _diskCacheSelector ?? "default"); //Ready for exefs profiles
 
             return processContext;
         }

src/Ryujinx.HLE/HOS/ModLoader.cs

@@ -5,6 +5,7 @@ using LibHac.FsSystem;
 using LibHac.Loader;
 using LibHac.Tools.FsSystem;
 using LibHac.Tools.FsSystem.RomFs;
+using LibHac.Util;
 using Ryujinx.Common.Configuration;
 using Ryujinx.Common.Logging;
 using Ryujinx.Common.Utilities;
@@ -18,6 +19,7 @@ using System.Collections.Specialized;
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Security.Cryptography;
 using LazyFile = Ryujinx.HLE.HOS.Services.Fs.FileSystemProxy.LazyFile;
 using Path = System.IO.Path;
 
@@ -583,6 +585,7 @@ namespace Ryujinx.HLE.HOS
             public BitVector32 Stubs;
             public BitVector32 Replaces;
             public MetaLoader Npdm;
+            public string Hash;
 
             public bool Modified => (Stubs.Data | Replaces.Data) != 0;
         }
@@ -593,8 +596,11 @@ namespace Ryujinx.HLE.HOS
             {
                 Stubs = new BitVector32(),
                 Replaces = new BitVector32(),
+                Hash = null,
             };
 
+            string tempHash = string.Empty;
+
             if (!_appMods.TryGetValue(applicationId, out ModCache mods) || mods.ExefsDirs.Count == 0)
             {
                 return modLoadResult;
@@ -630,8 +636,16 @@ namespace Ryujinx.HLE.HOS
 
                         modLoadResult.Replaces[1 << i] = true;
 
-                        nsos[i] = new NsoExecutable(nsoFile.OpenRead().AsStorage(), nsoName);
+                        using (FileStream stream = nsoFile.OpenRead())
+                        {
+                            nsos[i] = new NsoExecutable(stream.AsStorage(), nsoName);
                             Logger.Info?.Print(LogClass.ModLoader, $"NSO '{nsoName}' replaced");
+                            using (MD5 md5 = MD5.Create())
+                            {
+                                stream.Seek(0, SeekOrigin.Begin);
+                                tempHash += BitConverter.ToString(md5.ComputeHash(stream)).Replace("-", "").ToLowerInvariant();
+                            }
+                        }
                     }
 
                     modLoadResult.Stubs[1 << i] |= File.Exists(Path.Combine(mod.Path.FullName, nsoName + StubExtension));
@@ -663,6 +677,14 @@ namespace Ryujinx.HLE.HOS
                 }
             }
 
+            if (!string.IsNullOrEmpty(tempHash))
+            {
+                using (MD5 md5 = MD5.Create())
+                {
+                    modLoadResult.Hash += BitConverter.ToString(md5.ComputeHash(tempHash.ToBytes())).Replace("-", "").ToLowerInvariant();
+                }
+            }
+
             return modLoadResult;
         }
 

src/Ryujinx.HLE/Loaders/Processes/Extensions/FileSystemExtensions.cs

@@ -82,13 +82,6 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
             // Apply Nsos patches.
             device.Configuration.VirtualFileSystem.ModLoader.ApplyNsoPatches(programId, nsoExecutables);
 
-            // Don't use PTC if ExeFS files have been replaced.
-            bool enablePtc = device.System.EnablePtc && !modLoadResult.Modified;
-            if (!enablePtc)
-            {
-                Logger.Warning?.Print(LogClass.Ptc, "Detected unsupported ExeFs modifications. PTC disabled.");
-            }
-
             string programName = string.Empty;
 
             if (!isHomebrew && programId > 0x010000000000FFFF)
@@ -115,7 +108,8 @@ namespace Ryujinx.HLE.Loaders.Processes.Extensions
                 device.System.KernelContext,
                 metaLoader,
                 nacpData,
-                enablePtc,
+                device.System.EnablePtc,
+                modLoadResult.Hash,
                 true,
                 programName,
                 metaLoader.GetProgramId(),

src/Ryujinx.HLE/Loaders/Processes/ProcessLoader.cs

@@ -212,6 +212,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                                                                        dummyExeFs.GetNpdm(),
                                                                        nacpData,
                                                                        diskCacheEnabled: false,
+                                                                       diskCacheSelector: string.Empty,
                                                                        allowCodeMemoryForJit: true,
                                                                        programName,
                                                                        programId,

src/Ryujinx.HLE/Loaders/Processes/ProcessLoaderHelper.cs

@@ -186,6 +186,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                 string.Empty,
                 string.Empty,
                 false,
+                string.Empty,
                 codeAddress,
                 codeSize);
 
@@ -226,6 +227,7 @@ namespace Ryujinx.HLE.Loaders.Processes
             MetaLoader metaLoader,
             BlitStruct<ApplicationControlProperty> applicationControlProperties,
             bool diskCacheEnabled,
+            string diskCacheSelector,
             bool allowCodeMemoryForJit,
             string name,
             ulong programId,
@@ -374,6 +376,7 @@ namespace Ryujinx.HLE.Loaders.Processes
                 $"{programId:x16}",
                 displayVersion,
                 diskCacheEnabled,
+                diskCacheSelector,
                 codeStart,
                 codeSize);