Better debugging for freezing

Torch.Server/TorchServer.cs

@@ -3,12 +3,14 @@ using Sandbox.Engine.Utils;
 using Sandbox.Game;
 using Sandbox.Game.World;
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Security.Principal;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Xml.Serialization.GeneratedAssembly;
@@ -194,16 +196,62 @@ namespace Torch.Server
             ((TorchServer)state).Invoke(() => mre.Set());
             if (!mre.WaitOne(TimeSpan.FromSeconds(Instance.Config.TickTimeout)))
             {
-                var mainThread = MySandboxGame.Static.UpdateThread;
+                Log.Error(DumpFrozenThread(MySandboxGame.Static.UpdateThread));
-                if (mainThread.IsAlive)
+                throw new TimeoutException($"Server watchdog detected that the server was frozen for at least {((TorchServer)state).Config.TickTimeout} seconds.");
-                    mainThread.Suspend();
-                var stackTrace = new StackTrace(mainThread, true);
-                throw new TimeoutException($"Server watchdog detected that the server was frozen for at least {((TorchServer)state).Config.TickTimeout} seconds.\n{stackTrace}");
             }
 
             Log.Debug("Server watchdog responded");
         }
 
+        private static string DumpFrozenThread(Thread thread, int traces = 3, int pause = 5000)
+        {
+            var stacks = new List<string>(traces);
+            var totalSize = 0;
+            for (var i = 0; i < traces; i++)
+            {
+                string dump = DumpStack(thread).ToString();
+                totalSize += dump.Length;
+                stacks.Add(dump);
+                Thread.Sleep(pause);
+            }
+            string commonPrefix = StringUtils.CommonSuffix(stacks);
+            // Advance prefix to include the line terminator.
+            commonPrefix = commonPrefix.Substring(commonPrefix.IndexOf('\n') + 1);
+
+            var result = new StringBuilder(totalSize - (stacks.Count - 1) * commonPrefix.Length);
+            result.AppendLine($"Frozen thread dump {thread.Name}");
+            result.AppendLine("Common prefix:").AppendLine(commonPrefix);
+            for (var i = 0; i < stacks.Count; i++)
+                if (stacks[i].Length > commonPrefix.Length)
+                {
+                    result.AppendLine($"Suffix {i}");
+                    result.AppendLine(stacks[i].Substring(0, stacks[i].Length - commonPrefix.Length));
+                }
+            return result.ToString();
+        }
+
+        private static StackTrace DumpStack(Thread thread)
+        {
+            try
+            {
+                thread.Suspend();
+            }
+            catch
+            {
+                // ignored
+            }
+            var stack = new StackTrace(thread, true);
+            try
+            {
+                thread.Resume();
+            }
+            catch
+            {
+                // ignored
+            }
+            return stack;
+        }
+
         /// <inheritdoc />
         public override void Stop()
         {
@@ -253,28 +301,32 @@ namespace Torch.Server
         /// <param name="callerId">Caller of the save operation</param>
         private void SaveCompleted(SaveGameStatus statusCode, long callerId = 0)
         {
+            string response = null;
             switch (statusCode)
             {
                 case SaveGameStatus.Success:
                     Log.Info("Save completed.");
-                    // TODO
+                    response = "Saved game.";
-//                    Multiplayer.SendMessage("Saved game.", playerId: callerId);
                     break;
                 case SaveGameStatus.SaveInProgress:
                     Log.Error("Save failed, a save is already in progress.");
-//                    Multiplayer.SendMessage("Save failed, a save is already in progress.", playerId: callerId, font: MyFontEnum.Red);
+                    response = "Save failed, a save is already in progress.";
                     break;
                 case SaveGameStatus.GameNotReady:
                     Log.Error("Save failed, game was not ready.");
-//                    Multiplayer.SendMessage("Save failed, game was not ready.", playerId: callerId, font: MyFontEnum.Red);
+                    response = "Save failed, game was not ready.";
                     break;
                 case SaveGameStatus.TimedOut:
                     Log.Error("Save failed, save timed out.");
-//                    Multiplayer.SendMessage("Save failed, save timed out.", playerId: callerId, font: MyFontEnum.Red);
+                    response = "Save failed, save timed out.";
                     break;
                 default:
                     break;
             }
+            if (MySession.Static.Players.TryGetPlayerId(callerId, out MyPlayer.PlayerId result))
+            {
+                Managers.GetManager<IChatManagerServer>()?.SendMessageAsOther("Server", response, statusCode == SaveGameStatus.Success ? MyFontEnum.Green : MyFontEnum.Red, result.SteamId);
+            }
         }
     }
 }

Torch/Torch.csproj

@@ -208,6 +208,7 @@
     <Compile Include="Plugins\PluginManifest.cs" />
     <Compile Include="Utils\Reflection.cs" />
     <Compile Include="Managers\ScriptingManager.cs" />
+    <Compile Include="Utils\StringUtils.cs" />
     <Compile Include="Utils\TorchAssemblyResolver.cs" />
     <Compile Include="Utils\ReflectedManager.cs" />
     <Compile Include="Session\TorchSessionManager.cs" />

Torch/Utils/StringUtils.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Torch.Utils
+{
+    /// <summary>
+    /// Utility methods for strings
+    /// </summary>
+    public static class StringUtils
+    {
+        /// <summary>
+        /// Determines a common prefix for the given set of strings
+        /// </summary>
+        /// <param name="set">Set of strings</param>
+        /// <returns>Common prefix</returns>
+        public static string CommonPrefix(IEnumerable<string> set)
+        {
+            StringBuilder builder = null;
+            foreach (string other in set)
+            {
+                if (builder == null)
+                    builder = new StringBuilder(other);
+                if (builder.Length > other.Length)
+                    builder.Remove(other.Length, builder.Length - other.Length);
+                for (var i = 0; i < builder.Length; i++)
+                    if (builder[i] != other[i])
+                    {
+                        builder.Remove(i, builder.Length - i);
+                        break;
+                    }
+            }
+            return builder?.ToString() ?? "";
+        }
+
+        /// <summary>
+        /// Determines a common suffix for the given set of strings
+        /// </summary>
+        /// <param name="set">Set of strings</param>
+        /// <returns>Common suffix</returns>
+        public static string CommonSuffix(IEnumerable<string> set)
+        {
+            StringBuilder builder = null;
+            foreach (string other in set)
+            {
+                if (builder == null)
+                    builder = new StringBuilder(other);
+                if (builder.Length > other.Length)
+                    builder.Remove(0, builder.Length - other.Length);
+                for (var i = 0; i < builder.Length; i++)
+                {
+                    if (builder[builder.Length - 1 - i] != other[other.Length - 1 - i])
+                    {
+                        builder.Remove(0, builder.Length - i);
+                        break;
+                    }
+                }
+            }
+            return builder?.ToString() ?? "";
+        }
+    }
+}

Torch/Utils/TorchLauncher.cs

@@ -29,19 +29,7 @@ namespace Torch.Utils
             var allPaths = new HashSet<string> { exePath };
             foreach (string other in binaryPaths)
                 allPaths.Add(other.ToLower().Replace('/', '\\'));
-
+            var pathPrefix = StringUtils.CommonPrefix(allPaths);
-            var path = new StringBuilder(allPaths.First());
-            foreach (string other in binaryPaths)
-            {
-                if (path.Length > other.Length)
-                    path.Remove(other.Length, path.Length - other.Length);
-                for (var i = 0; i < path.Length; i++)
-                    if (path[i] != other[i])
-                    {
-                        path.Remove(i, path.Length - i);
-                        break;
-                    }
-            }
             AppDomain.CurrentDomain.AppendPrivatePath(String.Join(Path.PathSeparator.ToString(), allPaths));
             AppDomain.CurrentDomain.SetData(TorchKey, true);
             AppDomain.CurrentDomain.ExecuteAssemblyByName(entryPoint, args);
@@ -50,7 +38,7 @@ namespace Torch.Utils
             // exclude application base from probing
             var setup = new AppDomainSetup
             {
-                ApplicationBase = path.ToString(),
+                ApplicationBase = pathPrefix.ToString(),
                 PrivateBinPathProbe = "",
                 PrivateBinPath = string.Join(";", allPaths)
             };