Add Early Warning Feature to WatchDog

Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.

This will help diagnose what was going on in that time before the crash.

paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch

@@ -822,7 +822,7 @@
      protected void runServer() {
          try {
              if (!this.initServer()) {
-@@ -727,9 +1116,15 @@
+@@ -727,9 +1116,16 @@
              }
  
              this.nextTickTimeNanos = Util.getNanos();
@@ -831,6 +831,7 @@
              this.status = this.buildServerStatus();
  
 +            // Spigot start
++            org.spigotmc.WatchdogThread.hasStarted = true; // Paper
 +            Arrays.fill( this.recentTps, 20 );
 +            // Paper start - further improve server tick loop
 +            long tickSection = Util.getNanos();
@@ -839,7 +840,7 @@
              while (this.running) {
                  long i;
  
-@@ -744,11 +1139,30 @@
+@@ -744,11 +1140,30 @@
                      if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) {
                          long k = j / i;
  
@@ -870,7 +871,7 @@
  
                  boolean flag = i == 0L;
  
-@@ -757,6 +1171,8 @@
+@@ -757,6 +1172,8 @@
                      this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount);
                  }
  
@@ -879,7 +880,7 @@
                  this.nextTickTimeNanos += i;
  
                  try {
-@@ -830,6 +1246,13 @@
+@@ -830,6 +1247,13 @@
                      this.services.profileCache().clearExecutor();
                  }
  
@@ -893,7 +894,7 @@
                  this.onServerExit();
              }
  
-@@ -889,9 +1312,16 @@
+@@ -889,9 +1313,16 @@
      }
  
      private boolean haveTime() {
@@ -911,7 +912,7 @@
      public static boolean throwIfFatalException() {
          RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
  
-@@ -903,7 +1333,7 @@
+@@ -903,7 +1334,7 @@
      }
  
      public static void setFatalException(RuntimeException exception) {
@@ -920,7 +921,7 @@
      }
  
      @Override
-@@ -977,7 +1407,7 @@
+@@ -977,7 +1408,7 @@
          }
      }
  
@@ -929,7 +930,7 @@
          Profiler.get().incrementCounter("runTask");
          super.doRunTask(ticktask);
      }
-@@ -1025,6 +1455,7 @@
+@@ -1025,6 +1456,7 @@
      }
  
      public void tickServer(BooleanSupplier shouldKeepTicking) {
@@ -937,7 +938,7 @@
          long i = Util.getNanos();
          int j = this.pauseWhileEmptySeconds() * 20;
  
-@@ -1041,6 +1472,7 @@
+@@ -1041,6 +1473,7 @@
                      this.autoSave();
                  }
  
@@ -945,7 +946,7 @@
                  this.tickConnection();
                  return;
              }
-@@ -1055,12 +1487,13 @@
+@@ -1055,12 +1488,13 @@
          }
  
          --this.ticksUntilAutosave;
@@ -960,7 +961,7 @@
          gameprofilerfiller.push("tallying");
          long k = Util.getNanos() - i;
          int l = this.tickCount % 100;
-@@ -1074,7 +1507,7 @@
+@@ -1074,7 +1508,7 @@
      }
  
      private void autoSave() {
@@ -969,7 +970,7 @@
          MinecraftServer.LOGGER.debug("Autosave started");
          ProfilerFiller gameprofilerfiller = Profiler.get();
  
-@@ -1123,7 +1556,7 @@
+@@ -1123,7 +1557,7 @@
      private ServerStatus buildServerStatus() {
          ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus();
  
@@ -978,7 +979,7 @@
      }
  
      private ServerStatus.Players buildPlayerStatus() {
-@@ -1133,7 +1566,7 @@
+@@ -1133,7 +1567,7 @@
          if (this.hidesOnlinePlayers()) {
              return new ServerStatus.Players(i, list.size(), List.of());
          } else {
@@ -987,7 +988,7 @@
              ObjectArrayList<GameProfile> objectarraylist = new ObjectArrayList(j);
              int k = Mth.nextInt(this.random, 0, list.size() - j);
  
-@@ -1154,24 +1587,43 @@
+@@ -1154,24 +1588,43 @@
          this.getPlayerList().getPlayers().forEach((entityplayer) -> {
              entityplayer.connection.suspendFlushing();
          });
@@ -1031,7 +1032,7 @@
  
              gameprofilerfiller.push("tick");
  
-@@ -1186,6 +1638,7 @@
+@@ -1186,6 +1639,7 @@
  
              gameprofilerfiller.pop();
              gameprofilerfiller.pop();
@@ -1039,7 +1040,7 @@
          }
  
          gameprofilerfiller.popPush("connection");
-@@ -1265,8 +1718,24 @@
+@@ -1265,8 +1719,24 @@
      @Nullable
      public ServerLevel getLevel(ResourceKey<Level> key) {
          return (ServerLevel) this.levels.get(key);
@@ -1064,7 +1065,7 @@
      public Set<ResourceKey<Level>> levelKeys() {
          return this.levels.keySet();
      }
-@@ -1296,7 +1765,7 @@
+@@ -1296,7 +1766,7 @@
  
      @DontObfuscate
      public String getServerModName() {
@@ -1073,7 +1074,7 @@
      }
  
      public SystemReport fillSystemReport(SystemReport details) {
-@@ -1347,7 +1816,7 @@
+@@ -1347,7 +1817,7 @@
  
      @Override
      public void sendSystemMessage(Component message) {
@@ -1082,7 +1083,7 @@
      }
  
      public KeyPair getKeyPair() {
-@@ -1481,10 +1950,20 @@
+@@ -1481,10 +1951,20 @@
  
      @Override
      public String getMotd() {
@@ -1104,7 +1105,7 @@
          this.motd = motd;
      }
  
-@@ -1507,7 +1986,7 @@
+@@ -1507,7 +1987,7 @@
      }
  
      public ServerConnectionListener getConnection() {
@@ -1113,7 +1114,7 @@
      }
  
      public boolean isReady() {
-@@ -1634,11 +2113,11 @@
+@@ -1634,11 +2114,11 @@
  
      public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
          CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@@ -1127,7 +1128,7 @@
          }, this).thenCompose((immutablelist) -> {
              MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
              List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
-@@ -1654,6 +2133,7 @@
+@@ -1654,6 +2134,7 @@
          }).thenAcceptAsync((minecraftserver_reloadableresources) -> {
              this.resources.close();
              this.resources = minecraftserver_reloadableresources;
@@ -1135,7 +1136,7 @@
              this.packRepository.setSelected(dataPacks);
              WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
  
-@@ -1952,7 +2432,7 @@
+@@ -1952,7 +2433,7 @@
              final List<String> list = Lists.newArrayList();
              final GameRules gamerules = this.getGameRules();
  
@@ -1144,7 +1145,7 @@
                  @Override
                  public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
                      list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
-@@ -2058,7 +2538,7 @@
+@@ -2058,7 +2539,7 @@
              try {
                  label51:
                  {
@@ -1153,7 +1154,7 @@
  
                      try {
                          arraylist = Lists.newArrayList(NativeModuleLister.listModules());
-@@ -2108,6 +2588,21 @@
+@@ -2108,6 +2589,21 @@
  
      }
  
@@ -1175,7 +1176,7 @@
      private ProfilerFiller createProfiler() {
          if (this.willStartRecordingMetrics) {
              this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
-@@ -2225,18 +2720,24 @@
+@@ -2225,18 +2721,24 @@
      }
  
      public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) {