diff --git a/patches/server/Fix-and-optimise-world-force-upgrading.patch b/patches/server/Fix-and-optimise-world-force-upgrading.patch deleted file mode 100644 index 10b41f5308..0000000000 --- a/patches/server/Fix-and-optimise-world-force-upgrading.patch +++ /dev/null @@ -1,395 +0,0 @@ -From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 -From: Spottedleaf -Date: Thu, 20 May 2021 07:02:22 -0700 -Subject: [PATCH] Fix and optimise world force upgrading - -The WorldUpgrader class was incorrectly modified by -CB. It will store an IChunkLoader instance for all -dimension types in the world, but obviously with how -CB shifts around worlds only one dimension type exists -per world. But this would be OK if CB did this -change correctly. All IChunkLoader instances -will point to the same regionfiles. And all -IChunkLoader instances are going to be read from. - -This problem hasn't really been reported because -it relies on the persistent legacy data to be converted -as well to cause corruption. Why? Because the legacy -data is also shared, it will result in different -outputs from conversion (as once conversion for legacy -persistent data takes place, it is REMOVED - so the next -convert will _not_ have the data). Which means different -sizes on disk. Which means different regionfile sector -allocations. Which means there are 3 different possible -regionfile sector allocations in memory, and none of them -are going to be correct. - -I've fixed this by writing a world upgrader suited to -CB's changes to world folder format. It was brain dead -easy to add threading, so I did. - -== AT == -public net.minecraft.util.worldupdate.WorldUpgrader REGEX - -diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java -new file mode 100644 -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 ---- /dev/null -+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java -@@ -0,0 +0,0 @@ -+package io.papermc.paper.world; -+ -+import com.mojang.datafixers.DataFixer; -+import com.mojang.serialization.MapCodec; -+import net.minecraft.SharedConstants; -+import net.minecraft.core.RegistryAccess; -+import net.minecraft.core.registries.Registries; -+import net.minecraft.nbt.CompoundTag; -+import net.minecraft.resources.ResourceKey; -+import net.minecraft.util.worldupdate.WorldUpgrader; -+import net.minecraft.world.level.ChunkPos; -+import net.minecraft.world.level.Level; -+import net.minecraft.world.level.chunk.ChunkGenerator; -+import net.minecraft.world.level.chunk.storage.ChunkStorage; -+import net.minecraft.world.level.chunk.storage.RegionFileStorage; -+import net.minecraft.world.level.chunk.storage.RegionStorageInfo; -+import net.minecraft.world.level.dimension.LevelStem; -+import net.minecraft.world.level.storage.DimensionDataStorage; -+import net.minecraft.world.level.storage.LevelStorageSource; -+import org.apache.logging.log4j.LogManager; -+import org.apache.logging.log4j.Logger; -+import java.io.File; -+import java.io.IOException; -+import java.text.DecimalFormat; -+import java.util.Optional; -+import java.util.concurrent.ExecutorService; -+import java.util.concurrent.Executors; -+import java.util.concurrent.ThreadFactory; -+import java.util.concurrent.atomic.AtomicInteger; -+import java.util.concurrent.atomic.AtomicLong; -+import java.util.function.Supplier; -+ -+public class ThreadedWorldUpgrader { -+ -+ private static final Logger LOGGER = LogManager.getLogger(); -+ -+ private final ResourceKey dimensionType; -+ private final String worldName; -+ private final File worldDir; -+ private final ExecutorService threadPool; -+ private final DataFixer dataFixer; -+ private final RegistryAccess registryLookup; -+ private final Optional>> generatorKey; -+ private final boolean removeCaches; -+ private final boolean recreateRegionFiles; // TODO -+ -+ public ThreadedWorldUpgrader(final ResourceKey dimensionType, final String worldName, final File worldDir, final int threads, -+ final DataFixer dataFixer, final RegistryAccess registryLookup, final Optional>> generatorKey, -+ final boolean removeCaches, final boolean recreateRegionFiles) { -+ this.dimensionType = dimensionType; -+ this.worldName = worldName; -+ this.worldDir = worldDir; -+ this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() { -+ private final AtomicInteger threadCounter = new AtomicInteger(); -+ -+ @Override -+ public Thread newThread(final Runnable run) { -+ final Thread ret = new Thread(run); -+ -+ ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement()); -+ ret.setUncaughtExceptionHandler((thread, throwable) -> { -+ LOGGER.fatal("Error upgrading world", throwable); -+ }); -+ -+ return ret; -+ } -+ }); -+ this.dataFixer = dataFixer; -+ this.registryLookup = registryLookup; -+ this.generatorKey = generatorKey; -+ this.removeCaches = removeCaches; -+ this.recreateRegionFiles = recreateRegionFiles; -+ } -+ -+ public void convert() { -+ final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile(); -+ final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer, this.registryLookup); -+ -+ final File regionFolder = new File(worldFolder, "region"); -+ -+ LOGGER.info("Force upgrading {}", this.worldName); -+ LOGGER.info("Counting regionfiles for {}", this.worldName); -+ final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> { -+ return WorldUpgrader.REGEX.matcher(name).matches(); -+ }); -+ if (regionFiles == null) { -+ LOGGER.info("Found no regionfiles to convert for world {}", this.worldName); -+ return; -+ } -+ LOGGER.info("Found {} regionfiles to convert", regionFiles.length); -+ LOGGER.info("Starting conversion now for world {}", this.worldName); -+ -+ // Only used for profiling, let's fill it anyways just in case -+ final RegionStorageInfo storageInfo = new RegionStorageInfo( -+ this.worldName, -+ ResourceKey.create(Registries.DIMENSION, this.dimensionType.location()), -+ "region" -+ ); -+ -+ final WorldInfo info = new WorldInfo(() -> worldPersistentData, -+ new ChunkStorage(storageInfo, regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); -+ -+ long expectedChunks = (long)regionFiles.length * (32L * 32L); -+ -+ for (final File regionFile : regionFiles) { -+ final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath()); -+ if (regionPos == null) { -+ expectedChunks -= (32L * 32L); -+ continue; -+ } -+ -+ this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5)); -+ } -+ this.threadPool.shutdown(); -+ -+ final DecimalFormat format = new DecimalFormat("#0.00"); -+ -+ final long start = System.nanoTime(); -+ -+ while (!this.threadPool.isTerminated()) { -+ final long current = info.convertedChunks.get(); -+ -+ LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks); -+ -+ try { -+ Thread.sleep(1000L); -+ } catch (final InterruptedException ignore) {} -+ } -+ -+ final long end = System.nanoTime(); -+ -+ try { -+ info.loader.close(); -+ } catch (final IOException ex) { -+ LOGGER.fatal("Failed to close chunk loader", ex); -+ } -+ LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)", -+ (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0)); -+ } -+ -+ private static final class WorldInfo { -+ -+ public final Supplier persistentDataSupplier; -+ public final ChunkStorage loader; -+ public final boolean removeCaches; -+ public final ResourceKey worldKey; -+ public final Optional>> generatorKey; -+ public final AtomicLong convertedChunks = new AtomicLong(); -+ public final AtomicLong modifiedChunks = new AtomicLong(); -+ -+ private WorldInfo(final Supplier persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches, -+ final ResourceKey worldKey, Optional>> generatorKey) { -+ this.persistentDataSupplier = persistentDataSupplier; -+ this.loader = loader; -+ this.removeCaches = removeCaches; -+ this.worldKey = worldKey; -+ this.generatorKey = generatorKey; -+ } -+ } -+ -+ private static final class ConvertTask implements Runnable { -+ -+ private final WorldInfo worldInfo; -+ private final int regionX; -+ private final int regionZ; -+ -+ public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) { -+ this.worldInfo = worldInfo; -+ this.regionX = regionX; -+ this.regionZ = regionZ; -+ } -+ -+ @Override -+ public void run() { -+ final int regionCX = this.regionX << 5; -+ final int regionCZ = this.regionZ << 5; -+ -+ final Supplier persistentDataSupplier = this.worldInfo.persistentDataSupplier; -+ final ChunkStorage loader = this.worldInfo.loader; -+ final boolean removeCaches = this.worldInfo.removeCaches; -+ final ResourceKey worldKey = this.worldInfo.worldKey; -+ -+ for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) { -+ for (int cx = regionCX; cx < (regionCX + 32); ++cx) { -+ final ChunkPos chunkPos = new ChunkPos(cx, cz); -+ try { -+ // no need to check the coordinate of the chunk, the regionfilecache does that for us -+ -+ CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null); -+ -+ if (chunkNBT == null) { -+ continue; -+ } -+ -+ final int versionBefore = ChunkStorage.getVersion(chunkNBT); -+ -+ chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null); -+ -+ boolean modified = versionBefore < SharedConstants.getCurrentVersion().getDataVersion().getVersion(); -+ -+ if (removeCaches) { -+ final CompoundTag level = chunkNBT.getCompound("Level"); -+ modified |= level.contains("Heightmaps"); -+ level.remove("Heightmaps"); -+ modified |= level.contains("isLightOn"); -+ level.remove("isLightOn"); -+ } -+ -+ if (modified) { -+ this.worldInfo.modifiedChunks.getAndIncrement(); -+ loader.write(chunkPos, chunkNBT); -+ } -+ } catch (final Exception ex) { -+ LOGGER.error("Error upgrading chunk {}", chunkPos, ex); -+ } finally { -+ this.worldInfo.convertedChunks.getAndIncrement(); -+ } -+ } -+ } -+ } -+ } -+} -diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/net/minecraft/server/Main.java -+++ b/src/main/java/net/minecraft/server/Main.java -@@ -0,0 +0,0 @@ public class Main { - return new WorldLoader.InitConfig(worldloader_d, Commands.CommandSelection.DEDICATED, serverPropertiesHandler.functionPermissionLevel); - } - -+ // Paper start - fix and optimise world upgrading -+ public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession, -+ DataFixer dataFixer, RegistryAccess registryLookup, Optional>> generatorKey, boolean removeCaches, boolean recreateRegionFiles) { -+ int threads = Runtime.getRuntime().availableProcessors() * 3 / 8; -+ final io.papermc.paper.world.ThreadedWorldUpgrader worldUpgrader = new io.papermc.paper.world.ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, registryLookup, generatorKey, removeCaches, recreateRegionFiles); -+ worldUpgrader.convert(); -+ } -+ // Paper end - fix and optimise world upgrading -+ - public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, RegistryAccess dynamicRegistryManager, boolean recreateRegionFiles) { - Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit - WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, dynamicRegistryManager, eraseCache, recreateRegionFiles); -diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/net/minecraft/server/MinecraftServer.java -+++ b/src/main/java/net/minecraft/server/MinecraftServer.java -@@ -0,0 +0,0 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop { -- return true; -- }, iregistrycustom_dimension, this.options.has("recreateRegionFiles")); -- } -+ // Paper - fix and optimise world upgrading; move down - - PrimaryLevelData iworlddataserver = worlddata; - boolean flag = worlddata.isDebugWorld(); -@@ -0,0 +0,0 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop worldKey = ResourceKey.create(Registries.DIMENSION, dimensionKey.location()); - - if (dimensionKey == LevelStem.OVERWORLD) { -diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/net/minecraft/world/level/Level.java -+++ b/src/main/java/net/minecraft/world/level/Level.java -@@ -0,0 +0,0 @@ public abstract class Level implements LevelAccessor, AutoCloseable { - public final Map explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions - public java.util.ArrayDeque redstoneUpdateInfos; // Paper - Faster redstone torch rapid clock removal; Move from Map in BlockRedstoneTorch to here - -+ // Paper start - fix and optimise world upgrading -+ // copied from below -+ public static ResourceKey getDimensionKey(DimensionType manager) { -+ return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryAccess().registryOrThrow(net.minecraft.core.registries.Registries.DIMENSION_TYPE).getResourceKey(manager).orElseThrow(() -> { -+ return new IllegalStateException("Unregistered dimension type: " + manager); -+ }); -+ } -+ // Paper end - fix and optimise world upgrading -+ - public CraftWorld getWorld() { - return this.world; - } -diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java -@@ -0,0 +0,0 @@ public class RegionFileStorage implements AutoCloseable { - } - - // Paper start -+ @Nullable -+ public static ChunkPos getRegionFileCoordinates(Path file) { -+ String fileName = file.getFileName().toString(); -+ if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { -+ return null; -+ } -+ -+ String[] split = fileName.split("\\."); -+ -+ if (split.length != 4) { -+ return null; -+ } -+ -+ try { -+ int x = Integer.parseInt(split[1]); -+ int z = Integer.parseInt(split[2]); -+ -+ return new ChunkPos(x << 5, z << 5); -+ } catch (NumberFormatException ex) { -+ return null; -+ } -+ } -+ - public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { - return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ())); - } -diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java -+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java -@@ -0,0 +0,0 @@ public final class CraftServer implements Server { - worlddata.checkName(name); - worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified()); - -- if (this.console.options.has("forceUpgrade")) { -- net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), this.console.options.has("eraseCache"), () -> true, iregistrycustom_dimension, this.console.options.has("recreateRegionFiles")); -- } -+ // Paper - fix and optimise world upgrading; move down - - long j = BiomeManager.obfuscateSeed(worlddata.worldGenOptions().seed()); // Paper - use world seed - List list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata)); -@@ -0,0 +0,0 @@ public final class CraftServer implements Server { - biomeProvider = generator.getDefaultBiomeProvider(worldInfo); - } - -+ // Paper start - fix and optimise world upgrading -+ if (this.console.options.has("forceUpgrade")) { -+ net.minecraft.server.Main.convertWorldButItWorks( -+ actualDimension, worldSession, DataFixers.getDataFixer(), iregistrycustom_dimension, worlddimension.generator().getTypeNameForDataFixer(), this.console.options.has("eraseCache"), this.console.options.has("recreateRegionFiles") -+ ); -+ } -+ // Paper end - fix and optimise world upgrading - ResourceKey worldKey; - String levelName = this.getServer().getProperties().levelName; - if (name.equals(levelName + "_nether")) { diff --git a/patches/server/Write-SavedData-IO-async.patch b/patches/server/Write-SavedData-IO-async.patch index 28c0d12d60..cde49336c4 100644 --- a/patches/server/Write-SavedData-IO-async.patch +++ b/patches/server/Write-SavedData-IO-async.patch @@ -5,26 +5,6 @@ Subject: [PATCH] Write SavedData IO async Co-Authored-By: Shane Freeder -diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java -index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 ---- a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java -+++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java -@@ -0,0 +0,0 @@ public class ThreadedWorldUpgrader { - } - - this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5)); -+ // Paper start - Write SavedData IO async -+ this.threadPool.execute(() -> { -+ try { -+ worldPersistentData.close(); -+ } catch (IOException exception) { -+ LOGGER.error("Failed to close persistent world data", exception); -+ } -+ }); -+ // Paper end - Write SavedData IO async - } - this.threadPool.shutdown(); - diff --git a/src/main/java/net/minecraft/server/level/ServerChunkCache.java b/src/main/java/net/minecraft/server/level/ServerChunkCache.java index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 --- a/src/main/java/net/minecraft/server/level/ServerChunkCache.java