Parallelize FastLoader second config pass

CHANGELOG.md

@@ -3,6 +3,7 @@
 ##### Unreleased
 **New/Improved patches**
 - Improved the **FastLoader** patch to reuse the initial GameDatabase directory tree during the second config pass while refreshing files and directories created or modified by `Startup.Instantly` addons. Avoids reparsing unchanged configs and saves several seconds in heavily modded installs.
+- Improved the **FastLoader** second config pass by refreshing GameData directories in parallel and using faster filesystem functions.
 
 **Bug Fixes**
 - **EditorAnimatedPartsShipModified** : Fixed a memory leak ([issue #396](https://github.com/KSPModdingLibs/KSPCommunityFixes/issues/396)) where listeners were not properly cleaned up.

KSPCommunityFixes/Performance/FastLoader.cs

@@ -166,7 +166,7 @@ internal class KSPCFFastLoader : MonoBehaviour
 
         private static Harmony assetAndPartLoaderHarmony;
         private static string AssetAndPartLoaderHarmonyID => typeof(KSPCFFastLoader).FullName + "AssetAndPartLoader";
-        
+
         private static Harmony expansionsLoaderHarmony;
         private static string ExpansionsLoaderHarmonyID => typeof(KSPCFFastLoader).FullName + "ExpansionsLoader";
 
@@ -782,95 +782,130 @@ private static bool TryRefreshRoot(UrlDir root, ConfigDirectory[] configDirector
             }
 
             DateTime recentConfigCutoffUtc = DateTime.UtcNow.AddSeconds(-Time.realtimeSinceStartup);
+
+            var directoriesToRefresh = new List<UrlDir>();
             for (int i = 0; i < root.children.Count; i++)
-                RefreshDirectoryRecursive(root.children[i], fileConfig, recentConfigCutoffUtc);
+            {
+                UrlDir directory = root.children[i];
+                directoriesToRefresh.Add(directory);
+                directoriesToRefresh.AddRange(directory.AllDirectories);
+            }
+
+            Parallel.For(0, directoriesToRefresh.Count,
+                i => RefreshDirectory(directoriesToRefresh[i], fileConfig, recentConfigCutoffUtc));
 
             return true;
         }
 
-        private static void RefreshDirectoryRecursive(UrlDir dir, ConfigFileType[] fileConfig, DateTime recentConfigCutoffUtc)
-        {
-            var directoryInfo = new DirectoryInfo(dir.path);
-            var existingFiles = new Dictionary<string, UrlFile>(dir.files.Count, StringComparer.Ordinal);
-            for (int i = 0; i < dir.files.Count; i++)
-                existingFiles[dir.files[i].fullPath] = dir.files[i];
+        // Not safe to create UrlFile on multiple threads
+        private static readonly object urlCreationLock = new object();
 
-            var refreshedFiles = new List<UrlFile>(dir.files.Count);
-            foreach (FileInfo file in directoryInfo.GetFiles())
+        private static void RefreshDirectory(UrlDir dir, ConfigFileType[] fileConfig, DateTime recentConfigCutoffUtc)
+        {
+            string[] entries;
+            try
             {
-                // Files changed by Startup.Instantly need a new UrlFile so config contents and
-                // metadata such as fileTime match the second stock directory scan.
-                if (!existingFiles.Remove(file.FullName, out UrlFile urlFile)
-                    || !CanReuseFile(urlFile, file, recentConfigCutoffUtc))
-                {
-                    urlFile = new UrlFile(dir, file);
-                }
-
-                urlFile.ConfigureFile(fileConfig);
-                refreshedFiles.Add(urlFile);
+                // Fastest way to get all files and directories in a directory with ksp's mono
+                entries = Directory.GetFileSystemEntries(dir.path);
+            }
+            catch (DirectoryNotFoundException)
+            {
+                return;
             }
 
-            dir.files.Clear();
-            dir.files.AddRange(refreshedFiles);
+            var existingFiles = new Dictionary<string, UrlFile>(dir.files.Count, StringComparer.Ordinal);
+            for (int i = 0; i < dir.files.Count; i++)
+                existingFiles[dir.files[i].fullPath] = dir.files[i];
 
             var existingChildren = new Dictionary<string, UrlDir>(dir.children.Count, StringComparer.Ordinal);
             for (int i = 0; i < dir.children.Count; i++)
                 existingChildren[dir.children[i].path] = dir.children[i];
 
+            var refreshedFiles = new List<UrlFile>(dir.files.Count);
             var refreshedChildren = new List<UrlDir>(dir.children.Count);
-            foreach (DirectoryInfo directory in directoryInfo.GetDirectories())
+
+            for (int i = 0; i < entries.Length; i++)
             {
-                if (ShouldSkipStockDirectory(directory.Name))
+                string entryPath = entries[i];
+                if (!TryGetFileStat(entryPath, out MonoIOStat stat))
                     continue;
 
-                if (!existingChildren.Remove(directory.FullName, out UrlDir childDir))
+                // If directory
+                if ((stat.fileAttributes & FileAttributes.Directory) != 0)
                 {
-                    childDir = new UrlDir(dir, directory);
+                    if (existingChildren.TryGetValue(entryPath, out UrlDir childDir))
+                    {
+                        refreshedChildren.Add(childDir);
+                        continue;
+                    }
+
+                    var directory = new DirectoryInfo(entryPath);
+                    if (ShouldSkipStockDirectory(directory.Name))
+                        continue;
+
+                    lock (urlCreationLock)
+                        childDir = new UrlDir(dir, directory);
+
                     foreach (UrlFile file in childDir.files)
                         file.ConfigureFile(fileConfig);
                     foreach (UrlFile file in childDir.AllFiles)
                         file.ConfigureFile(fileConfig);
+
+                    refreshedChildren.Add(childDir);
                 }
                 else
                 {
-                    RefreshDirectoryRecursive(childDir, fileConfig, recentConfigCutoffUtc);
-                }
+                    if (!existingFiles.TryGetValue(entryPath, out UrlFile urlFile)
+                        || !CanReuseFile(urlFile, stat, recentConfigCutoffUtc))
+                    {
+                        lock (urlCreationLock)
+                            urlFile = new UrlFile(dir, new FileInfo(entryPath));
+                    }
 
-                refreshedChildren.Add(childDir);
+                    urlFile.ConfigureFile(fileConfig);
+                    refreshedFiles.Add(urlFile);
+                }
             }
 
+            dir.files.Clear();
+            dir.files.AddRange(refreshedFiles);
             dir.children.Clear();
             dir.children.AddRange(refreshedChildren);
         }
 
-        private static bool CanReuseFile(UrlFile urlFile, FileInfo file, DateTime recentConfigCutoffUtc)
+        // Much faster than FileInfo
+        private static bool TryGetFileStat(string path, out MonoIOStat stat)
         {
-            if (urlFile.fileTime != GetLastWriteTime(file))
-                return false;
-
-            if (urlFile.fileType != FileType.Config)
+            if (MonoIO.GetFileStat(path, out stat, out MonoIOError error))
                 return true;
 
-            try
-            {
-                return file.LastWriteTimeUtc < recentConfigCutoffUtc
-                    && file.CreationTimeUtc < recentConfigCutoffUtc;
-            }
-            catch
+            if (error == MonoIOError.ERROR_FILE_NOT_FOUND
+                || error == MonoIOError.ERROR_PATH_NOT_FOUND
+                || error == MonoIOError.ERROR_NOT_READY)
             {
                 return false;
             }
+
+            throw MonoIO.GetException(path, error);
         }
 
-        private static DateTime GetLastWriteTime(FileInfo file)
+        private static bool CanReuseFile(UrlFile urlFile, MonoIOStat stat, DateTime recentConfigCutoffUtc)
         {
             try
             {
-                return file.LastWriteTime;
+                DateTime lastWriteTimeUtc = DateTime.FromFileTimeUtc(stat.LastWriteTime);
+                if (urlFile.fileTime != lastWriteTimeUtc.ToLocalTime())
+                    return false;
+
+                if (urlFile.fileType != FileType.Config)
+                    return true;
+
+                return lastWriteTimeUtc < recentConfigCutoffUtc
+                    && DateTime.FromFileTimeUtc(stat.CreationTime) < recentConfigCutoffUtc;
             }
             catch
             {
-                return DateTime.MinValue;
+                return false;
             }
         }