diff --git a/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch b/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch index 41df18c7dc..4eb13cea5b 100644 --- a/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch +++ b/paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch @@ -822,7 +822,7 @@ protected void runServer() { try { if (!this.initServer()) { -@@ -727,9 +1116,15 @@ +@@ -727,9 +1116,16 @@ } this.nextTickTimeNanos = Util.getNanos(); @@ -831,6 +831,7 @@ this.status = this.buildServerStatus(); + // Spigot start ++ org.spigotmc.WatchdogThread.hasStarted = true; // Paper + Arrays.fill( this.recentTps, 20 ); + // Paper start - further improve server tick loop + long tickSection = Util.getNanos(); @@ -839,7 +840,7 @@ while (this.running) { long i; -@@ -744,11 +1139,30 @@ +@@ -744,11 +1140,30 @@ if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) { long k = j / i; @@ -870,7 +871,7 @@ boolean flag = i == 0L; -@@ -757,6 +1171,8 @@ +@@ -757,6 +1172,8 @@ this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount); } @@ -879,7 +880,7 @@ this.nextTickTimeNanos += i; try { -@@ -830,6 +1246,13 @@ +@@ -830,6 +1247,13 @@ this.services.profileCache().clearExecutor(); } @@ -893,7 +894,7 @@ this.onServerExit(); } -@@ -889,9 +1312,16 @@ +@@ -889,9 +1313,16 @@ } private boolean haveTime() { @@ -911,7 +912,7 @@ public static boolean throwIfFatalException() { RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get(); -@@ -903,7 +1333,7 @@ +@@ -903,7 +1334,7 @@ } public static void setFatalException(RuntimeException exception) { @@ -920,7 +921,7 @@ } @Override -@@ -977,7 +1407,7 @@ +@@ -977,7 +1408,7 @@ } } @@ -929,7 +930,7 @@ Profiler.get().incrementCounter("runTask"); super.doRunTask(ticktask); } -@@ -1025,6 +1455,7 @@ +@@ -1025,6 +1456,7 @@ } public void tickServer(BooleanSupplier shouldKeepTicking) { @@ -937,7 +938,7 @@ long i = Util.getNanos(); int j = this.pauseWhileEmptySeconds() * 20; -@@ -1041,6 +1472,7 @@ +@@ -1041,6 +1473,7 @@ this.autoSave(); } @@ -945,7 +946,7 @@ this.tickConnection(); return; } -@@ -1055,12 +1487,13 @@ +@@ -1055,12 +1488,13 @@ } --this.ticksUntilAutosave; @@ -960,7 +961,7 @@ gameprofilerfiller.push("tallying"); long k = Util.getNanos() - i; int l = this.tickCount % 100; -@@ -1074,7 +1507,7 @@ +@@ -1074,7 +1508,7 @@ } private void autoSave() { @@ -969,7 +970,7 @@ MinecraftServer.LOGGER.debug("Autosave started"); ProfilerFiller gameprofilerfiller = Profiler.get(); -@@ -1123,7 +1556,7 @@ +@@ -1123,7 +1557,7 @@ private ServerStatus buildServerStatus() { ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus(); @@ -978,7 +979,7 @@ } private ServerStatus.Players buildPlayerStatus() { -@@ -1133,7 +1566,7 @@ +@@ -1133,7 +1567,7 @@ if (this.hidesOnlinePlayers()) { return new ServerStatus.Players(i, list.size(), List.of()); } else { @@ -987,7 +988,7 @@ ObjectArrayList objectarraylist = new ObjectArrayList(j); int k = Mth.nextInt(this.random, 0, list.size() - j); -@@ -1154,24 +1587,43 @@ +@@ -1154,24 +1588,43 @@ this.getPlayerList().getPlayers().forEach((entityplayer) -> { entityplayer.connection.suspendFlushing(); }); @@ -1031,7 +1032,7 @@ gameprofilerfiller.push("tick"); -@@ -1186,6 +1638,7 @@ +@@ -1186,6 +1639,7 @@ gameprofilerfiller.pop(); gameprofilerfiller.pop(); @@ -1039,7 +1040,7 @@ } gameprofilerfiller.popPush("connection"); -@@ -1265,8 +1718,24 @@ +@@ -1265,8 +1719,24 @@ @Nullable public ServerLevel getLevel(ResourceKey key) { return (ServerLevel) this.levels.get(key); @@ -1064,7 +1065,7 @@ public Set> levelKeys() { return this.levels.keySet(); } -@@ -1296,7 +1765,7 @@ +@@ -1296,7 +1766,7 @@ @DontObfuscate public String getServerModName() { @@ -1073,7 +1074,7 @@ } public SystemReport fillSystemReport(SystemReport details) { -@@ -1347,7 +1816,7 @@ +@@ -1347,7 +1817,7 @@ @Override public void sendSystemMessage(Component message) { @@ -1082,7 +1083,7 @@ } public KeyPair getKeyPair() { -@@ -1481,10 +1950,20 @@ +@@ -1481,10 +1951,20 @@ @Override public String getMotd() { @@ -1104,7 +1105,7 @@ this.motd = motd; } -@@ -1507,7 +1986,7 @@ +@@ -1507,7 +1987,7 @@ } public ServerConnectionListener getConnection() { @@ -1113,7 +1114,7 @@ } public boolean isReady() { -@@ -1634,11 +2113,11 @@ +@@ -1634,11 +2114,11 @@ public CompletableFuture reloadResources(Collection dataPacks) { CompletableFuture completablefuture = CompletableFuture.supplyAsync(() -> { @@ -1127,7 +1128,7 @@ }, this).thenCompose((immutablelist) -> { MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist); List> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess()); -@@ -1654,6 +2133,7 @@ +@@ -1654,6 +2134,7 @@ }).thenAcceptAsync((minecraftserver_reloadableresources) -> { this.resources.close(); this.resources = minecraftserver_reloadableresources; @@ -1135,7 +1136,7 @@ this.packRepository.setSelected(dataPacks); WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures()); -@@ -1952,7 +2432,7 @@ +@@ -1952,7 +2433,7 @@ final List list = Lists.newArrayList(); final GameRules gamerules = this.getGameRules(); @@ -1144,7 +1145,7 @@ @Override public > void visit(GameRules.Key key, GameRules.Type type) { list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key))); -@@ -2058,7 +2538,7 @@ +@@ -2058,7 +2539,7 @@ try { label51: { @@ -1153,7 +1154,7 @@ try { arraylist = Lists.newArrayList(NativeModuleLister.listModules()); -@@ -2108,6 +2588,21 @@ +@@ -2108,6 +2589,21 @@ } @@ -1175,7 +1176,7 @@ private ProfilerFiller createProfiler() { if (this.willStartRecordingMetrics) { this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> { -@@ -2225,18 +2720,24 @@ +@@ -2225,18 +2721,24 @@ } public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) { diff --git a/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch b/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch index 51e7437c4d..748c832eb4 100644 --- a/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch +++ b/paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch @@ -142,7 +142,7 @@ thread.setDaemon(true); thread.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler(DedicatedServer.LOGGER)); thread.start(); -@@ -126,13 +203,26 @@ +@@ -126,13 +203,27 @@ this.setPreventProxyConnections(dedicatedserverproperties.preventProxyConnections); this.setLocalIp(dedicatedserverproperties.serverIp); } @@ -156,6 +156,7 @@ + this.paperConfigurations.initializeGlobalConfiguration(this.registryAccess()); + this.paperConfigurations.initializeWorldDefaultsConfiguration(this.registryAccess()); + // Paper end - initialize global and world-defaults configuration ++ org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash); // Paper - start watchdog thread + io.papermc.paper.command.PaperCommands.registerCommands(this); // Paper - setup /paper command + com.destroystokyo.paper.Metrics.PaperMetrics.startMetrics(); // Paper - start metrics + com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // Paper - load version history now @@ -170,7 +171,7 @@ DedicatedServer.LOGGER.info("Default game type: {}", dedicatedserverproperties.gamemode); InetAddress inetaddress = null; -@@ -156,21 +246,34 @@ +@@ -156,21 +247,34 @@ return false; } @@ -208,7 +209,7 @@ this.debugSampleSubscriptionTracker = new DebugSampleSubscriptionTracker(this.getPlayerList()); this.tickTimeLogger = new RemoteSampleLogger(TpsDebugDimensions.values().length, this.debugSampleSubscriptionTracker, RemoteDebugSampleType.TICK_TIME); long i = Util.getNanos(); -@@ -178,13 +281,13 @@ +@@ -178,13 +282,13 @@ SkullBlockEntity.setup(this.services, this); GameProfileCache.setUsesAuthentication(this.usesAuthentication()); DedicatedServer.LOGGER.info("Preparing level \"{}\"", this.getLevelIdName()); @@ -224,7 +225,7 @@ } if (dedicatedserverproperties.enableQuery) { -@@ -197,7 +300,7 @@ +@@ -197,7 +301,7 @@ this.rconThread = RconThread.create(this); } @@ -233,7 +234,7 @@ Thread thread1 = new Thread(new ServerWatchdog(this)); thread1.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandlerWithName(DedicatedServer.LOGGER)); -@@ -213,7 +316,13 @@ +@@ -213,7 +317,13 @@ return true; } @@ -247,7 +248,7 @@ @Override public boolean isSpawningMonsters() { -@@ -293,6 +402,7 @@ +@@ -293,6 +403,7 @@ this.queryThreadGs4.stop(); } @@ -255,7 +256,7 @@ } @Override -@@ -302,8 +412,8 @@ +@@ -302,8 +413,8 @@ } @Override @@ -266,7 +267,7 @@ } public void handleConsoleInput(String command, CommandSourceStack commandSource) { -@@ -314,7 +424,15 @@ +@@ -314,7 +425,15 @@ while (!this.consoleInput.isEmpty()) { ConsoleInput servercommand = (ConsoleInput) this.consoleInput.remove(0); @@ -283,7 +284,7 @@ } } -@@ -383,7 +501,7 @@ +@@ -383,7 +502,7 @@ @Override public boolean isUnderSpawnProtection(ServerLevel world, BlockPos pos, Player player) { @@ -292,7 +293,7 @@ return false; } else if (this.getPlayerList().getOps().isEmpty()) { return false; -@@ -453,7 +571,11 @@ +@@ -453,7 +572,11 @@ public boolean enforceSecureProfile() { DedicatedServerProperties dedicatedserverproperties = this.getProperties(); @@ -305,7 +306,7 @@ } @Override -@@ -541,16 +663,52 @@ +@@ -541,16 +664,52 @@ @Override public String getPluginNames() { @@ -362,7 +363,7 @@ } public void storeUsingWhiteList(boolean useWhitelist) { -@@ -660,4 +818,15 @@ +@@ -660,4 +819,15 @@ } } } diff --git a/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java index 64742de1ec..65856f97a5 100644 --- a/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java +++ b/paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java @@ -965,6 +965,7 @@ public final class CraftServer implements Server { @Override public void reload() { + org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload this.reloadCount++; this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile()); this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile()); @@ -1057,6 +1058,7 @@ public final class CraftServer implements Server { this.enablePlugins(PluginLoadOrder.POSTWORLD); if (io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper != null) io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper.pluginsEnabled(); // Paper - Remap plugins this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD)); + org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload } @Override diff --git a/paper-server/src/main/java/org/spigotmc/SpigotConfig.java b/paper-server/src/main/java/org/spigotmc/SpigotConfig.java index b97ff4a9b6..85f3433860 100644 --- a/paper-server/src/main/java/org/spigotmc/SpigotConfig.java +++ b/paper-server/src/main/java/org/spigotmc/SpigotConfig.java @@ -225,7 +225,7 @@ public class SpigotConfig SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript ); SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) ); SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) ); - WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); + // WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); // Paper - moved to after paper config initialization } public static boolean bungee; diff --git a/paper-server/src/main/java/org/spigotmc/WatchdogThread.java b/paper-server/src/main/java/org/spigotmc/WatchdogThread.java index 81ccbe2533..ad282d3491 100644 --- a/paper-server/src/main/java/org/spigotmc/WatchdogThread.java +++ b/paper-server/src/main/java/org/spigotmc/WatchdogThread.java @@ -14,6 +14,10 @@ public class WatchdogThread extends Thread private static WatchdogThread instance; private long timeoutTime; private boolean restart; + private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting + private final long earlyWarningDelay; // Paper + public static volatile boolean hasStarted; // Paper + private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps private volatile long lastTick; private volatile boolean stopping; @@ -22,6 +26,8 @@ public class WatchdogThread extends Thread super( "Paper Watchdog Thread" ); this.timeoutTime = timeoutTime; this.restart = restart; + earlyWarningEvery = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningEvery, timeoutTime); // Paper + earlyWarningDelay = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningDelay, timeoutTime); // Paper } private static long monotonicMillis() @@ -61,9 +67,18 @@ public class WatchdogThread extends Thread while ( !this.stopping ) { // - if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable + // Paper start + Logger log = Bukkit.getServer().getLogger(); + long currentTime = WatchdogThread.monotonicMillis(); + if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable { - Logger log = Bukkit.getServer().getLogger(); + boolean isLongTimeout = currentTime > lastTick + timeoutTime; + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... + lastEarlyWarning = currentTime; + if (isLongTimeout) { + // Paper end log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" ); @@ -92,29 +107,45 @@ public class WatchdogThread extends Thread } } // Paper end + } else + { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---"); + log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump"); + } + // Paper end - Different message for short timeout log.log( Level.SEVERE, "------------------------------" ); log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log ); log.log( Level.SEVERE, "------------------------------" ); // + // Paper start - Only print full dump on long timeouts + if ( isLongTimeout ) + { log.log( Level.SEVERE, "Entire Thread Dump:" ); ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true ); for ( ThreadInfo thread : threads ) { WatchdogThread.dumpThread( thread, log ); } + } else { + log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---"); + } + log.log( Level.SEVERE, "------------------------------" ); + if ( isLongTimeout ) + { if ( this.restart && !MinecraftServer.getServer().hasStopped() ) { RestartCommand.restart(); } break; + } // Paper end } try { - sleep( 10000 ); + sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout } catch ( InterruptedException ex ) { this.interrupt();