PvE/Torch
3
0
Fork 0
Code Issues 1 Pull Requests Actions Releases 21 Wiki Activity

Better debugging for freezing

Browse Source
This commit is contained in:
Westin Miller
2017-10-05 12:56:16 -07:00
parent d709bf68dd
commit 9e81b6316f
4 changed files with 129 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
Torch.Server/TorchServer.cs
View File

@@ -3,12 +3,14 @@ using Sandbox.Engine.Utils;
using Sandbox.Game;
using Sandbox.Game.World;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security.Principal;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Xml.Serialization.GeneratedAssembly;
@@ -194,16 +196,62 @@ namespace Torch.Server
((TorchServer)state).Invoke(() => mre.Set());
if (!mre.WaitOne(TimeSpan.FromSeconds(Instance.Config.TickTimeout)))
{
var mainThread = MySandboxGame.Static.UpdateThread;
if (mainThread.IsAlive)
mainThread.Suspend();
var stackTrace = new StackTrace(mainThread, true);
throw new TimeoutException($"Server watchdog detected that the server was frozen for at least {((TorchServer)state).Config.TickTimeout} seconds.\n{stackTrace}");
Log.Error(DumpFrozenThread(MySandboxGame.Static.UpdateThread));
throw new TimeoutException($"Server watchdog detected that the server was frozen for at least {((TorchServer)state).Config.TickTimeout} seconds.");
}
Log.Debug("Server watchdog responded");
}
private static string DumpFrozenThread(Thread thread, int traces = 3, int pause = 5000)
{
var stacks = new List<string>(traces);
var totalSize = 0;
for (var i = 0; i < traces; i++)
{
string dump = DumpStack(thread).ToString();
totalSize += dump.Length;
stacks.Add(dump);
Thread.Sleep(pause);
}
string commonPrefix = StringUtils.CommonSuffix(stacks);
// Advance prefix to include the line terminator.
commonPrefix = commonPrefix.Substring(commonPrefix.IndexOf('\n') + 1);
var result = new StringBuilder(totalSize - (stacks.Count - 1) * commonPrefix.Length);
result.AppendLine($"Frozen thread dump {thread.Name}");
result.AppendLine("Common prefix:").AppendLine(commonPrefix);
for (var i = 0; i < stacks.Count; i++)
if (stacks[i].Length > commonPrefix.Length)
{
result.AppendLine($"Suffix {i}");
result.AppendLine(stacks[i].Substring(0, stacks[i].Length - commonPrefix.Length));
}
return result.ToString();
}
private static StackTrace DumpStack(Thread thread)
{
try
{
thread.Suspend();
}
catch
{
// ignored
}
var stack = new StackTrace(thread, true);
try
{
thread.Resume();
}
catch
{
// ignored
}
return stack;
}
/// <inheritdoc />
public override void Stop()
{
@@ -253,28 +301,32 @@ namespace Torch.Server
/// <param name="callerId">Caller of the save operation</param>
private void SaveCompleted(SaveGameStatus statusCode, long callerId = 0)
{
string response = null;
switch (statusCode)
{
case SaveGameStatus.Success:
Log.Info("Save completed.");
// TODO
// Multiplayer.SendMessage("Saved game.", playerId: callerId);
response = "Saved game.";
break;
case SaveGameStatus.SaveInProgress:
Log.Error("Save failed, a save is already in progress.");
// Multiplayer.SendMessage("Save failed, a save is already in progress.", playerId: callerId, font: MyFontEnum.Red);
response = "Save failed, a save is already in progress.";
break;
case SaveGameStatus.GameNotReady:
Log.Error("Save failed, game was not ready.");
// Multiplayer.SendMessage("Save failed, game was not ready.", playerId: callerId, font: MyFontEnum.Red);
response = "Save failed, game was not ready.";
break;
case SaveGameStatus.TimedOut:
Log.Error("Save failed, save timed out.");
// Multiplayer.SendMessage("Save failed, save timed out.", playerId: callerId, font: MyFontEnum.Red);
response = "Save failed, save timed out.";
break;
default:
break;
}
if (MySession.Static.Players.TryGetPlayerId(callerId, out MyPlayer.PlayerId result))
{
Managers.GetManager<IChatManagerServer>()?.SendMessageAsOther("Server", response, statusCode == SaveGameStatus.Success ? MyFontEnum.Green : MyFontEnum.Red, result.SteamId);
}
}
}
}