From 315943614573f0e1973bafc761c27207b8ea2b45 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 5 Dec 2016 23:51:09 -0500 Subject: reimplement assembly caching (#187) This commit ensures DLLs are copied to the cache directory only if they changed, to avoid breaking debugging support unless necessary. To support this change, the assembly hash file has been replaced with a more detailed JSON structure, which is used to determine whether the cache is up-to-date and whether to use the cached or original assembly. Some mods contain multiple DLLs, which must be kept together to prevent assembly resolution issues; to simplify that (and avoid orphaned cache entries), each mod now has its own separate cache. --- .../AssemblyRewriting/AssemblyTypeRewriter.cs | 6 +- .../Framework/AssemblyRewriting/CacheEntry.cs | 46 ++++++++ .../Framework/AssemblyRewriting/CachePaths.cs | 10 +- .../Framework/AssemblyRewriting/RewriteResult.cs | 49 +++++++++ .../Framework/ModAssemblyLoader.cs | 120 +++++++++++---------- src/StardewModdingAPI/Program.cs | 34 ++++-- src/StardewModdingAPI/StardewModdingAPI.csproj | 2 + 7 files changed, 193 insertions(+), 74 deletions(-) create mode 100644 src/StardewModdingAPI/Framework/AssemblyRewriting/CacheEntry.cs create mode 100644 src/StardewModdingAPI/Framework/AssemblyRewriting/RewriteResult.cs diff --git a/src/StardewModdingAPI/Framework/AssemblyRewriting/AssemblyTypeRewriter.cs b/src/StardewModdingAPI/Framework/AssemblyRewriting/AssemblyTypeRewriter.cs index 3459488e..9d4d6b11 100644 --- a/src/StardewModdingAPI/Framework/AssemblyRewriting/AssemblyTypeRewriter.cs +++ b/src/StardewModdingAPI/Framework/AssemblyRewriting/AssemblyTypeRewriter.cs @@ -54,7 +54,8 @@ namespace StardewModdingAPI.Framework.AssemblyRewriting /// Rewrite the types referenced by an assembly. /// The assembly to rewrite. - public void RewriteAssembly(AssemblyDefinition assembly) + /// Returns whether the assembly was modified. + public bool RewriteAssembly(AssemblyDefinition assembly) { ModuleDefinition module = assembly.Modules.Single(); // technically an assembly can have multiple modules, but none of the build tools (including MSBuild) support it; simplify by assuming one module @@ -71,7 +72,7 @@ namespace StardewModdingAPI.Framework.AssemblyRewriting } } if (!shouldRewrite) - return; + return false; // add target assembly references foreach (AssemblyNameReference target in this.AssemblyMap.TargetReferences.Values) @@ -117,6 +118,7 @@ namespace StardewModdingAPI.Framework.AssemblyRewriting } method.Body.OptimizeMacros(); } + return true; } diff --git a/src/StardewModdingAPI/Framework/AssemblyRewriting/CacheEntry.cs b/src/StardewModdingAPI/Framework/AssemblyRewriting/CacheEntry.cs new file mode 100644 index 00000000..3dfbc78c --- /dev/null +++ b/src/StardewModdingAPI/Framework/AssemblyRewriting/CacheEntry.cs @@ -0,0 +1,46 @@ +using System.IO; + +namespace StardewModdingAPI.Framework.AssemblyRewriting +{ + /// Represents cached metadata for a rewritten assembly. + internal class CacheEntry + { + /********* + ** Accessors + *********/ + /// The MD5 hash for the original assembly. + public readonly string Hash; + + /// The SMAPI version used to rewrite the assembly. + public readonly string ApiVersion; + + /// Whether to use the cached assembly instead of the original assembly. + public readonly bool UseCachedAssembly; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The MD5 hash for the original assembly. + /// The SMAPI version used to rewrite the assembly. + /// Whether to use the cached assembly instead of the original assembly. + public CacheEntry(string hash, string apiVersion, bool useCachedAssembly) + { + this.Hash = hash; + this.ApiVersion = apiVersion; + this.UseCachedAssembly = useCachedAssembly; + } + + /// Get whether the cache entry is up-to-date for the given assembly hash. + /// The paths for the cached assembly. + /// The MD5 hash of the original assembly. + /// The current SMAPI version. + public bool IsUpToDate(CachePaths paths, string hash, Version currentVersion) + { + return hash == this.Hash + && this.ApiVersion == currentVersion.ToString() + && (!this.UseCachedAssembly || File.Exists(paths.Assembly)); + } + } +} \ No newline at end of file diff --git a/src/StardewModdingAPI/Framework/AssemblyRewriting/CachePaths.cs b/src/StardewModdingAPI/Framework/AssemblyRewriting/CachePaths.cs index 17c4d188..18861873 100644 --- a/src/StardewModdingAPI/Framework/AssemblyRewriting/CachePaths.cs +++ b/src/StardewModdingAPI/Framework/AssemblyRewriting/CachePaths.cs @@ -12,8 +12,8 @@ namespace StardewModdingAPI.Framework.AssemblyRewriting /// The file path of the assembly file. public string Assembly { get; } - /// The file path containing the MD5 hash for the assembly. - public string Hash { get; } + /// The file path containing the assembly metadata. + public string Metadata { get; } /********* @@ -22,12 +22,12 @@ namespace StardewModdingAPI.Framework.AssemblyRewriting /// Construct an instance. /// The directory path which contains the assembly. /// The file path of the assembly file. - /// The file path containing the MD5 hash for the assembly. - public CachePaths(string directory, string assembly, string hash) + /// The file path containing the assembly metadata. + public CachePaths(string directory, string assembly, string metadata) { this.Directory = directory; this.Assembly = assembly; - this.Hash = hash; + this.Metadata = metadata; } } } \ No newline at end of file diff --git a/src/StardewModdingAPI/Framework/AssemblyRewriting/RewriteResult.cs b/src/StardewModdingAPI/Framework/AssemblyRewriting/RewriteResult.cs new file mode 100644 index 00000000..8f34bb20 --- /dev/null +++ b/src/StardewModdingAPI/Framework/AssemblyRewriting/RewriteResult.cs @@ -0,0 +1,49 @@ +namespace StardewModdingAPI.Framework.AssemblyRewriting +{ + /// Metadata about a preprocessed assembly. + internal class RewriteResult + { + /********* + ** Accessors + *********/ + /// The original assembly path. + public readonly string OriginalAssemblyPath; + + /// The cache paths. + public readonly CachePaths CachePaths; + + /// The rewritten assembly bytes. + public readonly byte[] AssemblyBytes; + + /// The MD5 hash for the original assembly. + public readonly string Hash; + + /// Whether to use the cached assembly instead of the original assembly. + public readonly bool UseCachedAssembly; + + /// Whether this data is newer than the cache. + public readonly bool IsNewerThanCache; + + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// + /// The cache paths. + /// The rewritten assembly bytes. + /// The MD5 hash for the original assembly. + /// Whether to use the cached assembly instead of the original assembly. + /// Whether this data is newer than the cache. + public RewriteResult(string originalAssemblyPath, CachePaths cachePaths, byte[] assemblyBytes, string hash, bool useCachedAssembly, bool isNewerThanCache) + { + this.OriginalAssemblyPath = originalAssemblyPath; + this.CachePaths = cachePaths; + this.Hash = hash; + this.AssemblyBytes = assemblyBytes; + this.UseCachedAssembly = useCachedAssembly; + this.IsNewerThanCache = isNewerThanCache; + } + } +} diff --git a/src/StardewModdingAPI/Framework/ModAssemblyLoader.cs b/src/StardewModdingAPI/Framework/ModAssemblyLoader.cs index 51018b0b..1ceb8ad2 100644 --- a/src/StardewModdingAPI/Framework/ModAssemblyLoader.cs +++ b/src/StardewModdingAPI/Framework/ModAssemblyLoader.cs @@ -1,9 +1,11 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Security.Cryptography; using Mono.Cecil; +using Newtonsoft.Json; using StardewModdingAPI.AssemblyRewriters; using StardewModdingAPI.Framework.AssemblyRewriting; @@ -15,8 +17,8 @@ namespace StardewModdingAPI.Framework /********* ** Properties *********/ - /// The directory in which to cache data. - private readonly string CacheDirPath; + /// The name of the directory containing a mod's cached data. + private readonly string CacheDirName; /// Metadata for mapping assemblies to the current . private readonly PlatformAssemblyMap AssemblyMap; @@ -32,74 +34,76 @@ namespace StardewModdingAPI.Framework ** Public methods *********/ /// Construct an instance. - /// The cache directory. + /// The name of the directory containing a mod's cached data. /// The current game platform. /// Encapsulates monitoring and logging. - public ModAssemblyLoader(string cacheDirPath, Platform targetPlatform, IMonitor monitor) + public ModAssemblyLoader(string cacheDirName, Platform targetPlatform, IMonitor monitor) { - this.CacheDirPath = cacheDirPath; + this.CacheDirName = cacheDirName; this.Monitor = monitor; this.AssemblyMap = Constants.GetAssemblyMap(targetPlatform); this.AssemblyTypeRewriter = new AssemblyTypeRewriter(this.AssemblyMap, monitor); } - /// Preprocess an assembly and cache the modified version. + /// Preprocess an assembly unless the cache is up to date. /// The assembly file path. - public void ProcessAssembly(string assemblyPath) + /// Returns the rewrite metadata for the preprocessed assembly. + public RewriteResult ProcessAssemblyUnlessCached(string assemblyPath) { // read assembly data - string assemblyFileName = Path.GetFileName(assemblyPath); - string assemblyDir = Path.GetDirectoryName(assemblyPath); byte[] assemblyBytes = File.ReadAllBytes(assemblyPath); - string hash = $"SMAPI {Constants.Version}|" + string.Join("", MD5.Create().ComputeHash(assemblyBytes).Select(p => p.ToString("X2"))); + string hash = string.Join("", MD5.Create().ComputeHash(assemblyBytes).Select(p => p.ToString("X2"))); - // check cache - CachePaths cachePaths = this.GetCacheInfo(assemblyPath); - bool canUseCache = File.Exists(cachePaths.Assembly) && File.Exists(cachePaths.Hash) && hash == File.ReadAllText(cachePaths.Hash); - - // process assembly if not cached - if (!canUseCache) + // get cached result if current + CachePaths cachePaths = this.GetCachePaths(assemblyPath); + { + CacheEntry cacheEntry = File.Exists(cachePaths.Metadata) ? JsonConvert.DeserializeObject(File.ReadAllText(cachePaths.Metadata)) : null; + if (cacheEntry != null && cacheEntry.IsUpToDate(cachePaths, hash, Constants.Version)) + return new RewriteResult(assemblyPath, cachePaths, assemblyBytes, cacheEntry.Hash, cacheEntry.UseCachedAssembly, isNewerThanCache: false); // no rewrite needed + } + this.Monitor.Log($"Preprocessing {Path.GetFileName(assemblyPath)} for compatibility...", LogLevel.Trace); + + // rewrite assembly + AssemblyDefinition assembly; + using (Stream readStream = new MemoryStream(assemblyBytes)) + assembly = AssemblyDefinition.ReadAssembly(readStream); + bool modified = this.AssemblyTypeRewriter.RewriteAssembly(assembly); + using (MemoryStream outStream = new MemoryStream()) { - this.Monitor.Log($"Loading {assemblyFileName} for the first time; preprocessing...", LogLevel.Trace); - - // read assembly definition - AssemblyDefinition assembly; - using (Stream readStream = new MemoryStream(assemblyBytes)) - assembly = AssemblyDefinition.ReadAssembly(readStream); - - // rewrite assembly to match platform - this.AssemblyTypeRewriter.RewriteAssembly(assembly); - - // write cache - using (MemoryStream outStream = new MemoryStream()) - { - // get assembly bytes - assembly.Write(outStream); - byte[] outBytes = outStream.ToArray(); - - // write assembly data - Directory.CreateDirectory(cachePaths.Directory); - File.WriteAllBytes(cachePaths.Assembly, outBytes); - File.WriteAllText(cachePaths.Hash, hash); - - // copy any mdb/pdb files - foreach (string path in Directory.GetFiles(assemblyDir, "*.mdb").Concat(Directory.GetFiles(assemblyDir, "*.pdb"))) - { - string filename = Path.GetFileName(path); - File.Copy(path, Path.Combine(cachePaths.Directory, filename), overwrite: true); - } - } + assembly.Write(outStream); + byte[] outBytes = outStream.ToArray(); + return new RewriteResult(assemblyPath, cachePaths, outBytes, hash, useCachedAssembly: modified, isNewerThanCache: true); } } - /// Load a preprocessed assembly. - /// The assembly file path. - public Assembly LoadCachedAssembly(string assemblyPath) + /// Write rewritten assembly metadata to the cache for a mod. + /// The rewrite results. + /// Whether to write all assemblies to the cache, even if they weren't modified. + /// There are no results to write, or the results are not all for the same directory. + public void WriteCache(IEnumerable results, bool forceCacheAssemblies) { - CachePaths cachePaths = this.GetCacheInfo(assemblyPath); - if (!File.Exists(cachePaths.Assembly)) - throw new InvalidOperationException($"The assembly {assemblyPath} doesn't exist in the preprocessed cache."); - return Assembly.UnsafeLoadFrom(cachePaths.Assembly); // unsafe load allows DLLs downloaded from the Internet without the user needing to 'unblock' them + results = results.ToArray(); + + // get cache directory + if (!results.Any()) + throw new InvalidOperationException("There are no assemblies to cache."); + if (results.Select(p => p.CachePaths.Directory).Distinct().Count() > 1) + throw new InvalidOperationException("The assemblies can't be cached together because they have different source directories."); + string cacheDir = results.Select(p => p.CachePaths.Directory).First(); + + // reset cache + if (Directory.Exists(cacheDir)) + Directory.Delete(cacheDir, recursive: true); + Directory.CreateDirectory(cacheDir); + + // cache all results + foreach (RewriteResult result in results) + { + CacheEntry cacheEntry = new CacheEntry(result.Hash, Constants.Version.ToString(), forceCacheAssemblies || result.UseCachedAssembly); + File.WriteAllText(result.CachePaths.Metadata, JsonConvert.SerializeObject(cacheEntry)); + if (forceCacheAssemblies || result.UseCachedAssembly) + File.WriteAllBytes(result.CachePaths.Assembly, result.AssemblyBytes); + } } /// Resolve an assembly from its name. @@ -124,13 +128,13 @@ namespace StardewModdingAPI.Framework *********/ /// Get the cache details for an assembly. /// The assembly file path. - private CachePaths GetCacheInfo(string assemblyPath) + private CachePaths GetCachePaths(string assemblyPath) { - string key = Path.GetFileNameWithoutExtension(assemblyPath); - string dirPath = Path.Combine(this.CacheDirPath, new DirectoryInfo(Path.GetDirectoryName(assemblyPath)).Name); - string cacheAssemblyPath = Path.Combine(dirPath, $"{key}.dll"); - string cacheHashPath = Path.Combine(dirPath, $"{key}.hash"); - return new CachePaths(dirPath, cacheAssemblyPath, cacheHashPath); + string fileName = Path.GetFileName(assemblyPath); + string dirPath = Path.Combine(Path.GetDirectoryName(assemblyPath), this.CacheDirName); + string cacheAssemblyPath = Path.Combine(dirPath, fileName); + string metadataPath = Path.Combine(dirPath, $"{fileName}.json"); + return new CachePaths(dirPath, cacheAssemblyPath, metadataPath); } } } diff --git a/src/StardewModdingAPI/Program.cs b/src/StardewModdingAPI/Program.cs index e648ed64..a46f7a3e 100644 --- a/src/StardewModdingAPI/Program.cs +++ b/src/StardewModdingAPI/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -12,6 +13,7 @@ using Newtonsoft.Json; using StardewModdingAPI.AssemblyRewriters; using StardewModdingAPI.Events; using StardewModdingAPI.Framework; +using StardewModdingAPI.Framework.AssemblyRewriting; using StardewModdingAPI.Inheritance; using StardewValley; using Monitor = StardewModdingAPI.Framework.Monitor; @@ -38,8 +40,8 @@ namespace StardewModdingAPI /// The full path to the folder containing mods. private static readonly string ModPath = Path.Combine(Constants.ExecutionPath, "Mods"); - /// The full path to the folder containing cached SMAPI data. - private static readonly string CachePath = Path.Combine(Program.ModPath, ".cache"); + /// The name of the folder containing a mod's cached assembly data. + private static readonly string CacheDirName = ".cache"; /// The log file to which to write messages. private static readonly LogFileManager LogFile = new LogFileManager(Constants.LogPath); @@ -134,7 +136,6 @@ namespace StardewModdingAPI Program.Monitor.Log("Loading SMAPI..."); Console.Title = Constants.ConsoleTitle; Program.VerifyPath(Program.ModPath); - Program.VerifyPath(Program.CachePath); Program.VerifyPath(Constants.LogDir); if (!File.Exists(Program.GameExecutablePath)) { @@ -304,7 +305,7 @@ namespace StardewModdingAPI Program.Monitor.Log("Loading mods..."); // get assembly loader - ModAssemblyLoader modAssemblyLoader = new ModAssemblyLoader(Program.CachePath, Program.TargetPlatform, Program.Monitor); + ModAssemblyLoader modAssemblyLoader = new ModAssemblyLoader(Program.CacheDirName, Program.TargetPlatform, Program.Monitor); AppDomain.CurrentDomain.AssemblyResolve += (sender, e) => modAssemblyLoader.ResolveAssembly(e.Name); // load mods @@ -401,14 +402,15 @@ namespace StardewModdingAPI } } - // preprocess mod assemblies + // preprocess mod assemblies for compatibility + var processedAssemblies = new List(); { bool succeeded = true; foreach (string assemblyPath in Directory.GetFiles(directory, "*.dll")) { try { - modAssemblyLoader.ProcessAssembly(assemblyPath); + processedAssemblies.Add(modAssemblyLoader.ProcessAssemblyUnlessCached(assemblyPath)); } catch (Exception ex) { @@ -420,13 +422,27 @@ namespace StardewModdingAPI if (!succeeded) continue; } + bool forceUseCachedAssembly = processedAssemblies.Any(p => p.UseCachedAssembly); // make sure DLLs are kept together for dependency resolution + if (processedAssemblies.Any(p => p.IsNewerThanCache)) + modAssemblyLoader.WriteCache(processedAssemblies, forceUseCachedAssembly); - // load assembly + // get entry assembly path + string mainAssemblyPath; + { + RewriteResult mainProcessedAssembly = processedAssemblies.FirstOrDefault(p => p.OriginalAssemblyPath == Path.Combine(directory, manifest.EntryDll)); + if (mainProcessedAssembly == null) + { + Program.Monitor.Log($"{errorPrefix}: the specified mod DLL does not exist.", LogLevel.Error); + continue; + } + mainAssemblyPath = forceUseCachedAssembly ? mainProcessedAssembly.CachePaths.Assembly : mainProcessedAssembly.OriginalAssemblyPath; + } + + // load entry assembly Assembly modAssembly; try { - string assemblyPath = Path.Combine(directory, manifest.EntryDll); - modAssembly = modAssemblyLoader.LoadCachedAssembly(assemblyPath); + modAssembly = Assembly.UnsafeLoadFrom(mainAssemblyPath); // unsafe load allows downloaded DLLs if (modAssembly.DefinedTypes.Count(x => x.BaseType == typeof(Mod)) == 0) { Program.Monitor.Log($"{errorPrefix}: the mod DLL does not contain an implementation of the 'Mod' class.", LogLevel.Error); diff --git a/src/StardewModdingAPI/StardewModdingAPI.csproj b/src/StardewModdingAPI/StardewModdingAPI.csproj index 96eb038e..8a827ace 100644 --- a/src/StardewModdingAPI/StardewModdingAPI.csproj +++ b/src/StardewModdingAPI/StardewModdingAPI.csproj @@ -154,8 +154,10 @@ + + -- cgit