diff --git a/CHANGELOG.md b/CHANGELOG.md index bf3aea9..2a5878c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ##### Unreleased **New/Improved patches** - Improved the **FastLoader** patch to reuse the initial GameDatabase directory tree during the second config pass while refreshing files and directories created or modified by `Startup.Instantly` addons. Avoids reparsing unchanged configs and saves several seconds in heavily modded installs. +- Improved the **FastLoader** second config pass by refreshing GameData directories in parallel and using faster filesystem functions. **Bug Fixes** - **EditorAnimatedPartsShipModified** : Fixed a memory leak ([issue #396](https://github.com/KSPModdingLibs/KSPCommunityFixes/issues/396)) where listeners were not properly cleaned up. diff --git a/KSPCommunityFixes/Performance/FastLoader.cs b/KSPCommunityFixes/Performance/FastLoader.cs index 4d37235..842b5c6 100644 --- a/KSPCommunityFixes/Performance/FastLoader.cs +++ b/KSPCommunityFixes/Performance/FastLoader.cs @@ -166,7 +166,7 @@ internal class KSPCFFastLoader : MonoBehaviour private static Harmony assetAndPartLoaderHarmony; private static string AssetAndPartLoaderHarmonyID => typeof(KSPCFFastLoader).FullName + "AssetAndPartLoader"; - + private static Harmony expansionsLoaderHarmony; private static string ExpansionsLoaderHarmonyID => typeof(KSPCFFastLoader).FullName + "ExpansionsLoader"; @@ -782,95 +782,130 @@ private static bool TryRefreshRoot(UrlDir root, ConfigDirectory[] configDirector } DateTime recentConfigCutoffUtc = DateTime.UtcNow.AddSeconds(-Time.realtimeSinceStartup); + + var directoriesToRefresh = new List(); for (int i = 0; i < root.children.Count; i++) - RefreshDirectoryRecursive(root.children[i], fileConfig, recentConfigCutoffUtc); + { + UrlDir directory = root.children[i]; + directoriesToRefresh.Add(directory); + directoriesToRefresh.AddRange(directory.AllDirectories); + } + + Parallel.For(0, directoriesToRefresh.Count, + i => RefreshDirectory(directoriesToRefresh[i], fileConfig, recentConfigCutoffUtc)); return true; } - private static void RefreshDirectoryRecursive(UrlDir dir, ConfigFileType[] fileConfig, DateTime recentConfigCutoffUtc) - { - var directoryInfo = new DirectoryInfo(dir.path); - var existingFiles = new Dictionary(dir.files.Count, StringComparer.Ordinal); - for (int i = 0; i < dir.files.Count; i++) - existingFiles[dir.files[i].fullPath] = dir.files[i]; + // Not safe to create UrlFile on multiple threads + private static readonly object urlCreationLock = new object(); - var refreshedFiles = new List(dir.files.Count); - foreach (FileInfo file in directoryInfo.GetFiles()) + private static void RefreshDirectory(UrlDir dir, ConfigFileType[] fileConfig, DateTime recentConfigCutoffUtc) + { + string[] entries; + try { - // Files changed by Startup.Instantly need a new UrlFile so config contents and - // metadata such as fileTime match the second stock directory scan. - if (!existingFiles.Remove(file.FullName, out UrlFile urlFile) - || !CanReuseFile(urlFile, file, recentConfigCutoffUtc)) - { - urlFile = new UrlFile(dir, file); - } - - urlFile.ConfigureFile(fileConfig); - refreshedFiles.Add(urlFile); + // Fastest way to get all files and directories in a directory with ksp's mono + entries = Directory.GetFileSystemEntries(dir.path); + } + catch (DirectoryNotFoundException) + { + return; } - dir.files.Clear(); - dir.files.AddRange(refreshedFiles); + var existingFiles = new Dictionary(dir.files.Count, StringComparer.Ordinal); + for (int i = 0; i < dir.files.Count; i++) + existingFiles[dir.files[i].fullPath] = dir.files[i]; var existingChildren = new Dictionary(dir.children.Count, StringComparer.Ordinal); for (int i = 0; i < dir.children.Count; i++) existingChildren[dir.children[i].path] = dir.children[i]; + var refreshedFiles = new List(dir.files.Count); var refreshedChildren = new List(dir.children.Count); - foreach (DirectoryInfo directory in directoryInfo.GetDirectories()) + + for (int i = 0; i < entries.Length; i++) { - if (ShouldSkipStockDirectory(directory.Name)) + string entryPath = entries[i]; + if (!TryGetFileStat(entryPath, out MonoIOStat stat)) continue; - if (!existingChildren.Remove(directory.FullName, out UrlDir childDir)) + // If directory + if ((stat.fileAttributes & FileAttributes.Directory) != 0) { - childDir = new UrlDir(dir, directory); + if (existingChildren.TryGetValue(entryPath, out UrlDir childDir)) + { + refreshedChildren.Add(childDir); + continue; + } + + var directory = new DirectoryInfo(entryPath); + if (ShouldSkipStockDirectory(directory.Name)) + continue; + + lock (urlCreationLock) + childDir = new UrlDir(dir, directory); + foreach (UrlFile file in childDir.files) file.ConfigureFile(fileConfig); foreach (UrlFile file in childDir.AllFiles) file.ConfigureFile(fileConfig); + + refreshedChildren.Add(childDir); } else { - RefreshDirectoryRecursive(childDir, fileConfig, recentConfigCutoffUtc); - } + if (!existingFiles.TryGetValue(entryPath, out UrlFile urlFile) + || !CanReuseFile(urlFile, stat, recentConfigCutoffUtc)) + { + lock (urlCreationLock) + urlFile = new UrlFile(dir, new FileInfo(entryPath)); + } - refreshedChildren.Add(childDir); + urlFile.ConfigureFile(fileConfig); + refreshedFiles.Add(urlFile); + } } + dir.files.Clear(); + dir.files.AddRange(refreshedFiles); dir.children.Clear(); dir.children.AddRange(refreshedChildren); } - private static bool CanReuseFile(UrlFile urlFile, FileInfo file, DateTime recentConfigCutoffUtc) + // Much faster than FileInfo + private static bool TryGetFileStat(string path, out MonoIOStat stat) { - if (urlFile.fileTime != GetLastWriteTime(file)) - return false; - - if (urlFile.fileType != FileType.Config) + if (MonoIO.GetFileStat(path, out stat, out MonoIOError error)) return true; - try - { - return file.LastWriteTimeUtc < recentConfigCutoffUtc - && file.CreationTimeUtc < recentConfigCutoffUtc; - } - catch + if (error == MonoIOError.ERROR_FILE_NOT_FOUND + || error == MonoIOError.ERROR_PATH_NOT_FOUND + || error == MonoIOError.ERROR_NOT_READY) { return false; } + + throw MonoIO.GetException(path, error); } - private static DateTime GetLastWriteTime(FileInfo file) + private static bool CanReuseFile(UrlFile urlFile, MonoIOStat stat, DateTime recentConfigCutoffUtc) { try { - return file.LastWriteTime; + DateTime lastWriteTimeUtc = DateTime.FromFileTimeUtc(stat.LastWriteTime); + if (urlFile.fileTime != lastWriteTimeUtc.ToLocalTime()) + return false; + + if (urlFile.fileType != FileType.Config) + return true; + + return lastWriteTimeUtc < recentConfigCutoffUtc + && DateTime.FromFileTimeUtc(stat.CreationTime) < recentConfigCutoffUtc; } catch { - return DateTime.MinValue; + return false; } }