From 92f8c1d1205750ad675560f1ebbfbf6554477b9a Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Thu, 15 May 2025 23:34:22 +0200 Subject: [PATCH 01/10] Experiment with only recompiling changed classes --- build.gradle | 5 + interfaces_test.json | 5 + .../actions/ApplySourceTransformAction.java | 2 +- .../runtime/actions/MergeZipsAction.java | 42 +++++++ .../actions/RecompileSourcesAction.java | 14 ++- .../RecompileSourcesActionWithECJ.java | 9 ++ .../RecompileSourcesActionWithJDK.java | 9 ++ .../actions/SelectSourcesToRecompile.java | 105 ++++++++++++++++++ .../runtime/cli/RunNeoFormCommand.java | 93 ++++++++++++---- .../graph/transforms/ReplaceNodeOutput.java | 43 ++++--- test_at.cfg | 35 ++++++ 11 files changed, 324 insertions(+), 38 deletions(-) create mode 100644 interfaces_test.json create mode 100644 src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java create mode 100644 src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java create mode 100644 test_at.cfg diff --git a/build.gradle b/build.gradle index 1ed8db9e..a1f0c283 100644 --- a/build.gradle +++ b/build.gradle @@ -179,6 +179,11 @@ idea { programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --write-result=compiled:build/minecraft.jar --write-result=clientResources:build/client-extra.jar --write-result=sources:build/minecraft-sources.jar" moduleRef(project, sourceSets.main) } + "Experiments"(Application) { + mainClass = mainClassName + programParameters = "run --verbose --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --interface-injection-data=interfaces_test.json --validated-access-transformer=test_at.cfg --write-result=node.injectModifiedClasses.output.output:build/injectModifiedClasses-output.jar" + moduleRef(project, sourceSets.main) + } "Run Neoform 1.21 (joined)"(Application) { mainClass = mainClassName programParameters = "run --dist joined --neoform net.neoforged:neoform:1.21-20240613.152323@zip --write-result=compiled:build/minecraft.jar --write-result=clientResources:build/client-extra.jar --write-result=sources:build/minecraft-sources.jar" diff --git a/interfaces_test.json b/interfaces_test.json new file mode 100644 index 00000000..c7fc996b --- /dev/null +++ b/interfaces_test.json @@ -0,0 +1,5 @@ +{ + "net/minecraft/world/item/Item": [ + "testproject/FunExtensions" + ] +} diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java index a6bd0490..9fad8a12 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java @@ -109,7 +109,7 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt for (var path : validatedAccessTransformers) { args.add("--access-transformer"); - args.add(environment.getPathArgument(path)); + args.add(environment.getPathArgument(path.toAbsolutePath())); // TODO: uniformize with the other calls in this class } for (var path : additionalAccessTransformers) { diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java new file mode 100644 index 00000000..f100bb37 --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java @@ -0,0 +1,42 @@ +package net.neoforged.neoform.runtime.actions; + +import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.zip.ZipException; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +// TODO: would be good to "merge" with MergeWithSourcesAction? +public class MergeZipsAction extends BuiltInAction { + @Override + public void run(ProcessingEnvironment environment) throws IOException, InterruptedException { + var classesFile = environment.getRequiredInputPath("classes"); + var extraClassesFile = environment.getRequiredInputPath("classes2"); + var output = environment.getOutputPath("output"); + + try (var os = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(output)))) { + + boolean first = true; + for (var sourceZip : List.of(classesFile, extraClassesFile)) { + try (var in = new ZipInputStream(new BufferedInputStream(Files.newInputStream(sourceZip)))) { + for (var entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) { + if (!first && entry.isDirectory()) { + // Skip directories in case they would be duplicated + // TODO: a bit crude + continue; + } + os.putNextEntry(entry); + in.transferTo(os); + os.closeEntry(); + } + } + first = false; + } + } + } +} diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java index dc2fac3e..08c337c5 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java @@ -12,8 +12,8 @@ public abstract class RecompileSourcesAction extends BuiltInAction implements ExecutionNodeAction { - private final ExtensibleClasspath classpath = new ExtensibleClasspath(); - private final ExtensibleClasspath sourcepath = new ExtensibleClasspath(); + private ExtensibleClasspath classpath = new ExtensibleClasspath(); + private ExtensibleClasspath sourcepath = new ExtensibleClasspath(); private int targetJavaVersion = 21; @Override @@ -51,10 +51,18 @@ public ExtensibleClasspath getClasspath() { return classpath; } + public void setClasspath(ExtensibleClasspath classpath) { + this.classpath = classpath; + } + public ExtensibleClasspath getSourcepath() { return sourcepath; } + public void setSourcepath(ExtensibleClasspath sourcepath) { + this.sourcepath = sourcepath; + } + public int getTargetJavaVersion() { return targetJavaVersion; } @@ -62,4 +70,6 @@ public int getTargetJavaVersion() { public void setTargetJavaVersion(int targetJavaVersion) { this.targetJavaVersion = targetJavaVersion; } + + public abstract RecompileSourcesAction copy(); } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java index 92c5e509..bd6bc572 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java @@ -212,6 +212,15 @@ public void computeCacheKey(CacheKeyBuilder ck) { ck.add("compiler type", "eclipse"); } + @Override + public RecompileSourcesAction copy() { + var ret = new RecompileSourcesActionWithECJ(); + ret.setClasspath(getClasspath().copy()); + ret.setSourcepath(getSourcepath().copy()); + ret.setTargetJavaVersion(getTargetJavaVersion()); + return ret; + } + static class ECJFilesystem extends FileSystem { protected ECJFilesystem(Classpath[] paths, String[] initialFileNames, boolean annotationsFromClasspath) { super(paths, initialFileNames, annotationsFromClasspath); diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java index 64c86212..6ef8052f 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java @@ -108,4 +108,13 @@ private List getCompilerOptions() { "-implicit:none" // Prevents source files from the source-path from being emitted ); } + + @Override + public RecompileSourcesAction copy() { + var ret = new RecompileSourcesActionWithJDK(); + ret.setClasspath(getClasspath().copy()); + ret.setSourcepath(getSourcepath().copy()); + ret.setTargetJavaVersion(getTargetJavaVersion()); + return ret; + } } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java new file mode 100644 index 00000000..06ff28df --- /dev/null +++ b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java @@ -0,0 +1,105 @@ +package net.neoforged.neoform.runtime.actions; + +import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +public class SelectSourcesToRecompile extends BuiltInAction { + @Override + public void run(ProcessingEnvironment environment) throws IOException, InterruptedException { + var originalSources = environment.getRequiredInputPath("originalSources"); + var originalClasses = environment.getRequiredInputPath("originalClasses"); + var transformedSources = environment.getRequiredInputPath("transformedSources"); + + var unchangedClasses = environment.getOutputPath("unchangedClasses"); + var changedSourcesOnly = environment.getOutputPath("changedSourcesOnly"); + + // Read all original sources to memory + var originalSourcesContents = new HashMap(); + try (var zf = new ZipFile(originalSources.toFile())) { + var entries = zf.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + try (var is = zf.getInputStream(entry)) { + originalSourcesContents.put(entry.getName(), is.readAllBytes()); + } + } + } + + // Copy unchanged sources to the output + var changedSourcePaths = new HashSet(); + try (var os = Files.newOutputStream(changedSourcesOnly); + var zos = new ZipOutputStream(os)) { + + try (var transformedZip = new ZipFile(transformedSources.toFile())) { + var entries = transformedZip.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + + try (var is = transformedZip.getInputStream(entry)) { + var data = is.readAllBytes(); + + if (!Arrays.equals(data, originalSourcesContents.get(entry.getName()))) { + changedSourcePaths.add(entry.getName()); + // Copy to output + var copiedEntry = new ZipEntry(entry.getName()); + copiedEntry.setMethod(entry.getMethod()); + zos.putNextEntry(copiedEntry); + zos.write(data); + zos.closeEntry(); + } + } + } + } + } + + // Copy unchanged classes + try (var os = Files.newOutputStream(unchangedClasses); + var zos = new ZipOutputStream(os)) { + + try (var originalZip = new ZipFile(originalClasses.toFile())) { + var entries = originalZip.entries(); + while (entries.hasMoreElements()) { + var entry = entries.nextElement(); + if (!entry.isDirectory()) { + // Remove trailing .class + var sourceName = entry.getName(); + if (sourceName.endsWith(".class")) { + sourceName = sourceName.substring(0, sourceName.length() - 6); + } + // Remove inner class suffix + sourceName = sourceName.split("\\$")[0]; + // Add trailing .java + sourceName += ".java"; + + if (changedSourcePaths.contains(sourceName)) { + // Skip this entry + continue; + } + } + + try (var is = originalZip.getInputStream(entry)) { + // Copy to output + var copiedEntry = new ZipEntry(entry.getName()); + zos.putNextEntry(copiedEntry); + is.transferTo(zos); + zos.closeEntry(); + } + } + } + } + } +} diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index 08654daa..4b1475cf 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -5,8 +5,10 @@ import net.neoforged.neoform.runtime.actions.InjectFromZipFileSource; import net.neoforged.neoform.runtime.actions.InjectZipContentAction; import net.neoforged.neoform.runtime.actions.MergeWithSourcesAction; +import net.neoforged.neoform.runtime.actions.MergeZipsAction; import net.neoforged.neoform.runtime.actions.PatchActionFactory; import net.neoforged.neoform.runtime.actions.RecompileSourcesAction; +import net.neoforged.neoform.runtime.actions.SelectSourcesToRecompile; import net.neoforged.neoform.runtime.actions.StripManifestDigestContentFilter; import net.neoforged.neoform.runtime.artifacts.ClasspathItem; import net.neoforged.neoform.runtime.config.neoforge.NeoForgeConfig; @@ -218,8 +220,6 @@ protected void runWithNeoFormEngine(NeoFormEngine engine, List cl engine.loadNeoFormData(neoFormDataPath, dist); } - applyAdditionalAccessTransformers(engine); - if (parchmentData != null) { var parchmentDataFile = artifactManager.get(parchmentData); Consumer jstConsumer = transformSources -> { @@ -237,36 +237,87 @@ protected void runWithNeoFormEngine(NeoFormEngine engine, List cl } } - if (!interfaceInjectionDataFiles.isEmpty()) { - var transformNode = getOrAddTransformSourcesNode(engine); - ((ApplySourceTransformAction) transformNode.action()).setInjectedInterfaces(interfaceInjectionDataFiles); - - // Add the stub source jar to the recomp classpath - engine.applyTransform(new ModifyAction<>( - "recompile", - RecompileSourcesAction.class, - action -> { - action.getSourcepath().add(ClasspathItem.of(transformNode.getRequiredOutput("stubs"))); - } - )); - } + applyAdditionalTransforms(engine); execute(engine); } /** - * Configure the engine to apply additional user-supplied access transformers to the game sources. + * Configure the engine to apply additional user-supplied access transformers and interfaces to the game sources. + * This is done by re-transforming the sources a second time, and only recompiling changed sources. */ - private void applyAdditionalAccessTransformers(NeoFormEngine engine) { + private void applyAdditionalTransforms(NeoFormEngine engine) { + if (additionalAccessTransformers.isEmpty() && validatedAccessTransformers.isEmpty() && interfaceInjectionDataFiles.isEmpty()) { + return; + } + + var graph = engine.getGraph(); + + // TODO: what if there is no transformSources node yet? + var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); + sourceTransform(action -> {}) + .make(applyAdditionalTransformsBuilder, graph.getRequiredOutput("transformSources", "output")); + var applyAdditionalTransforms = applyAdditionalTransformsBuilder.build(); + // TODO: ugly + var sourceTransformAction = (ApplySourceTransformAction) applyAdditionalTransforms.action(); + + new ReplaceNodeOutput( + "recompile", + "output", + (engine_, recompileOutput) -> { + var selectSourcesBuilder = graph.nodeBuilder("selectSourcesToRecompile"); + selectSourcesBuilder.inputFromNodeOutput("originalSources", "transformSources", "output"); + selectSourcesBuilder.input("originalClasses", recompileOutput.asInput()); + selectSourcesBuilder.input("transformedSources", applyAdditionalTransforms.getRequiredOutput("output").asInput()); + var unchangedClasses = selectSourcesBuilder.output("unchangedClasses", NodeOutputType.JAR, "Classes that were already compiled and whose corresponding sources did not change."); + var changedSourcesOnly = selectSourcesBuilder.output("changedSourcesOnly", NodeOutputType.JAR, "Sources that were changed and need to be recompiled."); + selectSourcesBuilder.action(new SelectSourcesToRecompile()); + var selectSources = selectSourcesBuilder.build(); + + var recompileModifiedSources = graph.nodeBuilder("recompileModifiedSources"); + recompileModifiedSources.input("sources", changedSourcesOnly.asInput()); + recompileModifiedSources.input("versionManifest", recompileOutput.getNode().getRequiredInput("versionManifest")); + var modifiedClasses = recompileModifiedSources.output("output", NodeOutputType.JAR, "Compiled minecraft sources that were changed."); + + var recompileAction = (RecompileSourcesAction) recompileOutput.getNode().action(); + // TODO: we need to ensure that the original classpath is not modified after this + RecompileSourcesAction recompileModifiedAction = recompileAction.copy(); + recompileModifiedAction.getClasspath().add(ClasspathItem.of(unchangedClasses)); + recompileModifiedSources.action(recompileModifiedAction); + recompileModifiedSources.build(); + + var injectBuilder = graph.nodeBuilder("injectModifiedClasses"); + injectBuilder.input("classes", unchangedClasses.asInput()); + injectBuilder.input("classes2", modifiedClasses.asInput()); + var output = injectBuilder.output("output", NodeOutputType.JAR, "Compiled Minecraft sources with additional changes merged in."); + injectBuilder.action(new MergeZipsAction()); + injectBuilder.build(); + + return new ReplaceNodeOutput.ReplacementResult(output, List.of(selectSources)); + }) + .apply(engine, graph); + if (!additionalAccessTransformers.isEmpty() || !validatedAccessTransformers.isEmpty()) { - var transformSources = getOrAddTransformSourcesAction(engine); - transformSources.setAdditionalAccessTransformers(additionalAccessTransformers.stream().map(Paths::get).toList()); - transformSources.setValidatedAccessTransformers(validatedAccessTransformers.stream().map(Paths::get).toList()); + sourceTransformAction.setAdditionalAccessTransformers(additionalAccessTransformers.stream().map(Paths::get).toList()); + sourceTransformAction.setValidatedAccessTransformers(validatedAccessTransformers.stream().map(Paths::get).toList()); if (validateAccessTransformers) { - transformSources.addArg("--access-transformer-validation=error"); + sourceTransformAction.addArg("--access-transformer-validation=error"); } } + + if (!interfaceInjectionDataFiles.isEmpty()) { + sourceTransformAction.setInjectedInterfaces(interfaceInjectionDataFiles); + + // Add the stub source jar to the recomp classpath + engine.applyTransform(new ModifyAction<>( + "recompileModifiedSources", + RecompileSourcesAction.class, + action -> { + action.getSourcepath().add(ClasspathItem.of(applyAdditionalTransforms.getRequiredOutput("stubs"))); + } + )); + } } private static NodeOutput createCompiledWithNeoForge(NeoFormEngine engine, ZipFile neoforgeClassesZip) { diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java index aff34444..a8b28592 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java @@ -2,27 +2,49 @@ import net.neoforged.neoform.runtime.engine.NeoFormEngine; import net.neoforged.neoform.runtime.graph.ExecutionGraph; +import net.neoforged.neoform.runtime.graph.ExecutionNode; import net.neoforged.neoform.runtime.graph.ExecutionNodeBuilder; import net.neoforged.neoform.runtime.graph.NodeOutput; +import java.util.List; + public final class ReplaceNodeOutput extends GraphTransform { private final String nodeId; private final String outputId; - private final String newNodeId; - private final NodeFactory nodeFactory; + private final GeneralizedNodeFactory nodeFactory; @FunctionalInterface public interface NodeFactory { NodeOutput make(ExecutionNodeBuilder builder, NodeOutput previousNodeOutput); } + // TODO: cursed + public record ReplacementResult(NodeOutput newOutput, List nodesToIgnore) {} + + @FunctionalInterface + public interface GeneralizedNodeFactory { + ReplacementResult make(NeoFormEngine engine, NodeOutput previousNodeOutput); + } + public ReplaceNodeOutput(String nodeId, String outputId, String newNodeId, NodeFactory nodeFactory) { this.nodeId = nodeId; this.outputId = outputId; - this.newNodeId = newNodeId; + this.nodeFactory = (engine, previousNodeOutput) -> { + var builder = engine.getGraph().nodeBuilder(newNodeId); + var newOutput = nodeFactory.make(builder, previousNodeOutput); + var newNode = builder.build(); + return new ReplacementResult(newOutput, List.of(newNode)); + }; + } + + public ReplaceNodeOutput(String nodeId, + String outputId, + GeneralizedNodeFactory nodeFactory) { + this.nodeId = nodeId; + this.outputId = outputId; this.nodeFactory = nodeFactory; } @@ -30,31 +52,24 @@ public String nodeId() { return nodeId; } - public String outputId() { - return outputId; - } - @Override public void apply(NeoFormEngine engine, ExecutionGraph graph) { var originalOutput = graph.getRequiredOutput(nodeId, outputId); - // Add the additional node - var builder = graph.nodeBuilder(newNodeId); - var newOutput = this.nodeFactory.make(builder, originalOutput); - var newNode = builder.build(); + var replacementResult = this.nodeFactory.make(engine, originalOutput); // Find all uses of the old output and replace them with our new output for (var node : graph.getNodes()) { - if (node != newNode) { + if (!replacementResult.nodesToIgnore.contains(node)) { for (var nodeInput : node.inputs().values()) { - nodeInput.replaceReferences(originalOutput, newOutput); + nodeInput.replaceReferences(originalOutput, replacementResult.newOutput); } } } for (var entry : graph.getResults().entrySet()) { if (entry.getValue() == originalOutput) { - entry.setValue(newOutput); + entry.setValue(replacementResult.newOutput); } } } diff --git a/test_at.cfg b/test_at.cfg new file mode 100644 index 00000000..67c50f85 --- /dev/null +++ b/test_at.cfg @@ -0,0 +1,35 @@ +# Used for GUI rendering +protected net.minecraft.client.gui.components.Tooltip (Lnet/minecraft/network/chat/Component;Lnet/minecraft/network/chat/Component;)V +public-f net.minecraft.client.gui.components.AbstractWidget render(Lnet/minecraft/client/gui/GuiGraphics;IIF)V + +# Used for menu interactions +public net.minecraft.world.inventory.AbstractContainerMenu createCarriedSlotAccess()Lnet/minecraft/world/entity/SlotAccess; + +# Used for block state properties +public net.minecraft.world.level.block.Blocks never(Lnet/minecraft/world/level/block/state/BlockState;Lnet/minecraft/world/level/BlockGetter;Lnet/minecraft/core/BlockPos;)Z + +# Used for ore registration +public net.minecraft.data.worldgen.placement.OrePlacements commonOrePlacement(ILnet/minecraft/world/level/levelgen/placement/PlacementModifier;)Ljava/util/List; + +# Used to render pipe highlight +public net.minecraft.client.renderer.LevelRenderer renderShape(Lcom/mojang/blaze3d/vertex/PoseStack;Lcom/mojang/blaze3d/vertex/VertexConsumer;Lnet/minecraft/world/phys/shapes/VoxelShape;DDDFFFF)V + +# Used to access all chunks +public net.minecraft.server.level.ChunkMap updatingChunkMap + +# Used for datagen +public-f net.minecraft.data.recipes.RecipeProvider getName()Ljava/lang/String; +# To convert shaped recipes to machine recipes +public net.minecraft.world.item.crafting.ShapedRecipe pattern +public net.minecraft.world.item.crafting.ShapedRecipe result +public net.minecraft.world.item.crafting.ShapedRecipePattern data + +# Used to register villager trades +public net.minecraft.world.entity.npc.VillagerTrades$EmeraldForItems +public net.minecraft.world.entity.npc.VillagerTrades$ItemsForEmeralds + +# Used for jetpack +public net.minecraft.server.network.ServerGamePacketListenerImpl aboveGroundTickCount + +# Used to delay attacks after a barrel interaction +public net.minecraft.client.multiplayer.MultiPlayerGameMode destroyDelay From 9666713c691f03198dd34a1cb521dae705bfa3ea Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Fri, 16 May 2025 00:04:23 +0200 Subject: [PATCH 02/10] Use NIO for faster source selection --- .../actions/SelectSourcesToRecompile.java | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java index 06ff28df..3b20e9ae 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java @@ -3,10 +3,12 @@ import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; import java.io.IOException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; @@ -66,38 +68,35 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt } } - // Copy unchanged classes - try (var os = Files.newOutputStream(unchangedClasses); - var zos = new ZipOutputStream(os)) { + // Copy unchanged classes, by first copying the whole zip then deleting unwanted entries using NIO. + // This is faster than working with ZipFile streams which inflate and deflate all the entries. + Files.copy(originalClasses, unchangedClasses); + try (var zfs = FileSystems.newFileSystem(unchangedClasses, Map.of("create", "false"))) { try (var originalZip = new ZipFile(originalClasses.toFile())) { var entries = originalZip.entries(); while (entries.hasMoreElements()) { var entry = entries.nextElement(); - if (!entry.isDirectory()) { - // Remove trailing .class - var sourceName = entry.getName(); - if (sourceName.endsWith(".class")) { - sourceName = sourceName.substring(0, sourceName.length() - 6); - } - // Remove inner class suffix - sourceName = sourceName.split("\\$")[0]; - // Add trailing .java - sourceName += ".java"; - - if (changedSourcePaths.contains(sourceName)) { - // Skip this entry - continue; - } + if (entry.isDirectory()) { + continue; + } + // Remove trailing .class + var sourceName = entry.getName(); + if (sourceName.endsWith(".class")) { + sourceName = sourceName.substring(0, sourceName.length() - 6); } + // Remove inner class suffix + sourceName = sourceName.split("\\$")[0]; + // Add trailing .java + sourceName += ".java"; - try (var is = originalZip.getInputStream(entry)) { - // Copy to output - var copiedEntry = new ZipEntry(entry.getName()); - zos.putNextEntry(copiedEntry); - is.transferTo(zos); - zos.closeEntry(); + if (!changedSourcePaths.contains(sourceName)) { + // Skip this entry + continue; } + + // Delete! + Files.delete(zfs.getPath(entry.getName())); } } } From 1e7b39dc766af7d98fb31006181ed655252d1d5a Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Fri, 16 May 2025 00:21:41 +0200 Subject: [PATCH 03/10] Use NIO trick once again --- .../runtime/actions/MergeZipsAction.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java index f100bb37..2ca9d2fe 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java @@ -3,15 +3,12 @@ import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.IOException; +import java.nio.file.FileSystems; import java.nio.file.Files; -import java.util.List; -import java.util.zip.ZipException; +import java.util.Map; import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; -// TODO: would be good to "merge" with MergeWithSourcesAction? public class MergeZipsAction extends BuiltInAction { @Override public void run(ProcessingEnvironment environment) throws IOException, InterruptedException { @@ -19,23 +16,21 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt var extraClassesFile = environment.getRequiredInputPath("classes2"); var output = environment.getOutputPath("output"); - try (var os = new ZipOutputStream(new BufferedOutputStream(Files.newOutputStream(output)))) { + // Copy the largest jar then use NIO to insert extra entries. + // This is faster than working with ZipFile streams which inflate and deflate all the entries. + Files.copy(classesFile, output); + try (var zfs = FileSystems.newFileSystem(output, Map.of("create", false))) { + var zfsRoot = zfs.getPath("/"); - boolean first = true; - for (var sourceZip : List.of(classesFile, extraClassesFile)) { - try (var in = new ZipInputStream(new BufferedInputStream(Files.newInputStream(sourceZip)))) { - for (var entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) { - if (!first && entry.isDirectory()) { - // Skip directories in case they would be duplicated - // TODO: a bit crude - continue; - } - os.putNextEntry(entry); - in.transferTo(os); - os.closeEntry(); + // Copy the extra classes to the output zip + try (var in = new ZipInputStream(new BufferedInputStream(Files.newInputStream(extraClassesFile)))) { + for (var entry = in.getNextEntry(); entry != null; entry = in.getNextEntry()) { + if (entry.isDirectory()) { + continue; } + var targetPath = zfsRoot.resolve(entry.getName()); + Files.copy(in, targetPath); } - first = false; } } } From a99a0a6de0f73b6c1bdb4a8ee393e09fa8f676a6 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sat, 9 Aug 2025 21:26:33 +0200 Subject: [PATCH 04/10] Post merge fix --- .../net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index 4b1475cf..7e46f535 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -255,7 +255,7 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { // TODO: what if there is no transformSources node yet? var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); - sourceTransform(action -> {}) + sourceTransform(engine, action -> {}) .make(applyAdditionalTransformsBuilder, graph.getRequiredOutput("transformSources", "output")); var applyAdditionalTransforms = applyAdditionalTransformsBuilder.build(); // TODO: ugly From 275f005c33d80e492715a0ed04db2f924303b589 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sat, 9 Aug 2025 21:29:03 +0200 Subject: [PATCH 05/10] Address TODO --- .../neoform/runtime/actions/ApplySourceTransformAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java index 9fad8a12..df914b1c 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/ApplySourceTransformAction.java @@ -109,12 +109,12 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt for (var path : validatedAccessTransformers) { args.add("--access-transformer"); - args.add(environment.getPathArgument(path.toAbsolutePath())); // TODO: uniformize with the other calls in this class + args.add(environment.getPathArgument(path.toAbsolutePath())); } for (var path : additionalAccessTransformers) { args.add("--access-transformer"); - args.add(environment.getPathArgument(path)); + args.add(environment.getPathArgument(path.toAbsolutePath())); } } From ceda3bb006217be86df0e921f825224c1bbae962 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sun, 10 Aug 2025 10:31:35 +0200 Subject: [PATCH 06/10] Fix stuff --- build.gradle | 2 +- ...Action.java => InjectModifiedClasses.java} | 2 +- .../actions/SelectSourcesToRecompile.java | 1 - .../runtime/cli/RunNeoFormCommand.java | 111 ++++++++++-------- .../neoform/runtime/graph/ExecutionNode.java | 11 ++ .../graph/transforms/ReplaceNodeOutput.java | 50 ++++---- test_at.cfg | 2 +- 7 files changed, 102 insertions(+), 77 deletions(-) rename src/main/java/net/neoforged/neoform/runtime/actions/{MergeZipsAction.java => InjectModifiedClasses.java} (96%) diff --git a/build.gradle b/build.gradle index a1f0c283..8bff0255 100644 --- a/build.gradle +++ b/build.gradle @@ -181,7 +181,7 @@ idea { } "Experiments"(Application) { mainClass = mainClassName - programParameters = "run --verbose --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --interface-injection-data=interfaces_test.json --validated-access-transformer=test_at.cfg --write-result=node.injectModifiedClasses.output.output:build/injectModifiedClasses-output.jar" + programParameters = "run --verbose --partial-recompile --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --interface-injection-data=interfaces_test.json --validated-access-transformer=test_at.cfg --write-result=compiled:build/compiled-output.jar" moduleRef(project, sourceSets.main) } "Run Neoform 1.21 (joined)"(Application) { diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/InjectModifiedClasses.java similarity index 96% rename from src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java rename to src/main/java/net/neoforged/neoform/runtime/actions/InjectModifiedClasses.java index 2ca9d2fe..acf017cd 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/MergeZipsAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/InjectModifiedClasses.java @@ -9,7 +9,7 @@ import java.util.Map; import java.util.zip.ZipInputStream; -public class MergeZipsAction extends BuiltInAction { +public class InjectModifiedClasses extends BuiltInAction { @Override public void run(ProcessingEnvironment environment) throws IOException, InterruptedException { var classesFile = environment.getRequiredInputPath("classes"); diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java index 3b20e9ae..e0bba8bb 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java @@ -72,7 +72,6 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt // This is faster than working with ZipFile streams which inflate and deflate all the entries. Files.copy(originalClasses, unchangedClasses); try (var zfs = FileSystems.newFileSystem(unchangedClasses, Map.of("create", "false"))) { - try (var originalZip = new ZipFile(originalClasses.toFile())) { var entries = originalZip.entries(); while (entries.hasMoreElements()) { diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index 7e46f535..b04d42e3 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -5,14 +5,13 @@ import net.neoforged.neoform.runtime.actions.InjectFromZipFileSource; import net.neoforged.neoform.runtime.actions.InjectZipContentAction; import net.neoforged.neoform.runtime.actions.MergeWithSourcesAction; -import net.neoforged.neoform.runtime.actions.MergeZipsAction; +import net.neoforged.neoform.runtime.actions.InjectModifiedClasses; import net.neoforged.neoform.runtime.actions.PatchActionFactory; import net.neoforged.neoform.runtime.actions.RecompileSourcesAction; import net.neoforged.neoform.runtime.actions.SelectSourcesToRecompile; import net.neoforged.neoform.runtime.actions.StripManifestDigestContentFilter; import net.neoforged.neoform.runtime.artifacts.ClasspathItem; import net.neoforged.neoform.runtime.config.neoforge.NeoForgeConfig; -import net.neoforged.neoform.runtime.config.neoform.NeoFormDistConfig; import net.neoforged.neoform.runtime.engine.NeoFormEngine; import net.neoforged.neoform.runtime.graph.ExecutionGraph; import net.neoforged.neoform.runtime.graph.ExecutionNode; @@ -84,6 +83,9 @@ static class SourceArtifacts { String neoforge; } + @CommandLine.Option(names = "--partial-recompile", description = "[EXPERIMENTAL] Enables partial recompilation for user-defined transforms") + boolean partialRecompile; + @Override protected void runWithNeoFormEngine(NeoFormEngine engine, List closables) throws IOException, InterruptedException { var artifactManager = engine.getArtifactManager(); @@ -253,49 +255,64 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { var graph = engine.getGraph(); - // TODO: what if there is no transformSources node yet? - var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); - sourceTransform(engine, action -> {}) - .make(applyAdditionalTransformsBuilder, graph.getRequiredOutput("transformSources", "output")); - var applyAdditionalTransforms = applyAdditionalTransformsBuilder.build(); - // TODO: ugly - var sourceTransformAction = (ApplySourceTransformAction) applyAdditionalTransforms.action(); - - new ReplaceNodeOutput( - "recompile", - "output", - (engine_, recompileOutput) -> { - var selectSourcesBuilder = graph.nodeBuilder("selectSourcesToRecompile"); - selectSourcesBuilder.inputFromNodeOutput("originalSources", "transformSources", "output"); - selectSourcesBuilder.input("originalClasses", recompileOutput.asInput()); - selectSourcesBuilder.input("transformedSources", applyAdditionalTransforms.getRequiredOutput("output").asInput()); - var unchangedClasses = selectSourcesBuilder.output("unchangedClasses", NodeOutputType.JAR, "Classes that were already compiled and whose corresponding sources did not change."); - var changedSourcesOnly = selectSourcesBuilder.output("changedSourcesOnly", NodeOutputType.JAR, "Sources that were changed and need to be recompiled."); - selectSourcesBuilder.action(new SelectSourcesToRecompile()); - var selectSources = selectSourcesBuilder.build(); - - var recompileModifiedSources = graph.nodeBuilder("recompileModifiedSources"); - recompileModifiedSources.input("sources", changedSourcesOnly.asInput()); - recompileModifiedSources.input("versionManifest", recompileOutput.getNode().getRequiredInput("versionManifest")); - var modifiedClasses = recompileModifiedSources.output("output", NodeOutputType.JAR, "Compiled minecraft sources that were changed."); - - var recompileAction = (RecompileSourcesAction) recompileOutput.getNode().action(); - // TODO: we need to ensure that the original classpath is not modified after this - RecompileSourcesAction recompileModifiedAction = recompileAction.copy(); - recompileModifiedAction.getClasspath().add(ClasspathItem.of(unchangedClasses)); - recompileModifiedSources.action(recompileModifiedAction); - recompileModifiedSources.build(); - - var injectBuilder = graph.nodeBuilder("injectModifiedClasses"); - injectBuilder.input("classes", unchangedClasses.asInput()); - injectBuilder.input("classes2", modifiedClasses.asInput()); - var output = injectBuilder.output("output", NodeOutputType.JAR, "Compiled Minecraft sources with additional changes merged in."); - injectBuilder.action(new MergeZipsAction()); - injectBuilder.build(); - - return new ReplaceNodeOutput.ReplacementResult(output, List.of(selectSources)); - }) - .apply(engine, graph); + ExecutionNode sourceTransformNode; + ApplySourceTransformAction sourceTransformAction; + String recompilationNodeName; + + if (partialRecompile) { + // If there is a source transform (Parchment or NeoForge's AT), its output is the baseline. + // Else it's the output of patch (NeoForm with no Parchment applied). + var startingSources = graph.getNode("transformSources") != null + ? graph.getRequiredOutput("transformSources", "output") + : graph.getRequiredOutput("patch", "output"); + var recompileOutput = graph.getRequiredOutput("recompile", "output"); + var recompileAction = (RecompileSourcesAction) recompileOutput.getNode().action(); + + // Second transform action for additional transforms + var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); + sourceTransform(engine, action -> {}) + .make(applyAdditionalTransformsBuilder, startingSources); + sourceTransformNode = applyAdditionalTransformsBuilder.build(); + sourceTransformAction = (ApplySourceTransformAction) sourceTransformNode.action(); + + // Split off sources that were modified by the second transform + var selectSourcesBuilder = graph.nodeBuilder("selectSourcesToRecompile"); + selectSourcesBuilder.input("originalSources", startingSources.asInput()); + selectSourcesBuilder.input("originalClasses", recompileOutput.asInput()); + selectSourcesBuilder.input("transformedSources", sourceTransformNode.getRequiredOutput("output").asInput()); + var unchangedClasses = selectSourcesBuilder.output("unchangedClasses", NodeOutputType.JAR, "Classes that were already compiled and whose corresponding sources did not change."); + var changedSourcesOnly = selectSourcesBuilder.output("changedSourcesOnly", NodeOutputType.JAR, "Sources that were changed and need to be recompiled."); + selectSourcesBuilder.action(new SelectSourcesToRecompile()); + var selectSources = selectSourcesBuilder.build(); + + // Recompile modified sources + var recompileModifiedSources = graph.nodeBuilder("recompileModifiedSources"); + recompileModifiedSources.input("sources", changedSourcesOnly.asInput()); + recompileModifiedSources.input("versionManifest", recompileOutput.getNode().getRequiredInput("versionManifest")); + var modifiedClasses = recompileModifiedSources.output("output", NodeOutputType.JAR, "Compiled minecraft sources that were changed."); + RecompileSourcesAction recompileModifiedAction = recompileAction.copy(); + recompileModifiedAction.getClasspath().add(ClasspathItem.of(unchangedClasses)); + recompileModifiedSources.action(recompileModifiedAction); + recompileModifiedSources.build(); + + // Recombine with the unmodified classes that were already recompiled before + var injectBuilder = graph.nodeBuilder("injectModifiedClasses"); + injectBuilder.input("classes", unchangedClasses.asInput()); + injectBuilder.input("classes2", modifiedClasses.asInput()); + var injectedOutput = injectBuilder.output("output", NodeOutputType.JAR, "Compiled Minecraft sources with additional changes merged in."); + injectBuilder.action(new InjectModifiedClasses()); + injectBuilder.build(); + + // Since we replace one node by many, we cannot use the ReplaceNodeOutput with the usual factory, + // but rather directly call this helper + ReplaceNodeOutput.replaceOutput(graph, recompileOutput, injectedOutput, selectSources); + + recompilationNodeName = "recompileModifiedSources"; + } else { + sourceTransformNode = getOrAddTransformSourcesNode(engine); + sourceTransformAction = (ApplySourceTransformAction) sourceTransformNode.action(); + recompilationNodeName = "recompile"; + } if (!additionalAccessTransformers.isEmpty() || !validatedAccessTransformers.isEmpty()) { sourceTransformAction.setAdditionalAccessTransformers(additionalAccessTransformers.stream().map(Paths::get).toList()); @@ -311,10 +328,10 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { // Add the stub source jar to the recomp classpath engine.applyTransform(new ModifyAction<>( - "recompileModifiedSources", + recompilationNodeName, RecompileSourcesAction.class, action -> { - action.getSourcepath().add(ClasspathItem.of(applyAdditionalTransforms.getRequiredOutput("stubs"))); + action.getSourcepath().add(ClasspathItem.of(sourceTransformNode.getRequiredOutput("stubs"))); } )); } diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java b/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java index 2af09fe3..462dd6bd 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java @@ -1,5 +1,6 @@ package net.neoforged.neoform.runtime.graph; +import net.neoforged.neoform.runtime.actions.ApplySourceTransformAction; import net.neoforged.neoform.runtime.utils.AnsiColor; import net.neoforged.neoform.runtime.utils.Logger; @@ -145,6 +146,16 @@ public ExecutionNodeAction action() { return action; } + // TODO: not sure yet if this is useful + public T actionAs(Class clazz) { + if (clazz.isInstance(action)) { + return (T) action; + } else { + throw new IllegalStateException("Node " + id + " has a different action type than expected. Expected: " + + clazz + " but got " + action.getClass()); + } + } + @Override public String toString() { return id; diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java index a8b28592..7ce6c907 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java @@ -11,40 +11,21 @@ public final class ReplaceNodeOutput extends GraphTransform { private final String nodeId; private final String outputId; - private final GeneralizedNodeFactory nodeFactory; + private final String newNodeId; + private final NodeFactory nodeFactory; @FunctionalInterface public interface NodeFactory { NodeOutput make(ExecutionNodeBuilder builder, NodeOutput previousNodeOutput); } - // TODO: cursed - public record ReplacementResult(NodeOutput newOutput, List nodesToIgnore) {} - - @FunctionalInterface - public interface GeneralizedNodeFactory { - ReplacementResult make(NeoFormEngine engine, NodeOutput previousNodeOutput); - } - public ReplaceNodeOutput(String nodeId, String outputId, String newNodeId, NodeFactory nodeFactory) { this.nodeId = nodeId; this.outputId = outputId; - this.nodeFactory = (engine, previousNodeOutput) -> { - var builder = engine.getGraph().nodeBuilder(newNodeId); - var newOutput = nodeFactory.make(builder, previousNodeOutput); - var newNode = builder.build(); - return new ReplacementResult(newOutput, List.of(newNode)); - }; - } - - public ReplaceNodeOutput(String nodeId, - String outputId, - GeneralizedNodeFactory nodeFactory) { - this.nodeId = nodeId; - this.outputId = outputId; + this.newNodeId = newNodeId; this.nodeFactory = nodeFactory; } @@ -52,24 +33,41 @@ public String nodeId() { return nodeId; } + public String outputId() { + return outputId; + } + @Override public void apply(NeoFormEngine engine, ExecutionGraph graph) { var originalOutput = graph.getRequiredOutput(nodeId, outputId); - var replacementResult = this.nodeFactory.make(engine, originalOutput); + // Add the additional node + var builder = graph.nodeBuilder(newNodeId); + var newOutput = this.nodeFactory.make(builder, originalOutput); + var newNode = builder.build(); // Find all uses of the old output and replace them with our new output + replaceOutput(graph, originalOutput, newOutput, newNode); + } + + /** + * General output -> output replacement. + * + * @param toIgnore node for which the {@code originalOutput} should not be replaced by the {@code newOutput}, + * because it is part of the replacement pipeline + */ + public static void replaceOutput(ExecutionGraph graph, NodeOutput originalOutput, NodeOutput newOutput, ExecutionNode toIgnore) { for (var node : graph.getNodes()) { - if (!replacementResult.nodesToIgnore.contains(node)) { + if (node != toIgnore) { for (var nodeInput : node.inputs().values()) { - nodeInput.replaceReferences(originalOutput, replacementResult.newOutput); + nodeInput.replaceReferences(originalOutput, newOutput); } } } for (var entry : graph.getResults().entrySet()) { if (entry.getValue() == originalOutput) { - entry.setValue(replacementResult.newOutput); + entry.setValue(newOutput); } } } diff --git a/test_at.cfg b/test_at.cfg index 67c50f85..cc3e105e 100644 --- a/test_at.cfg +++ b/test_at.cfg @@ -1,6 +1,6 @@ # Used for GUI rendering protected net.minecraft.client.gui.components.Tooltip (Lnet/minecraft/network/chat/Component;Lnet/minecraft/network/chat/Component;)V -public-f net.minecraft.client.gui.components.AbstractWidget render(Lnet/minecraft/client/gui/GuiGraphics;IIF)V +protected-f net.minecraft.client.gui.components.AbstractWidget render(Lnet/minecraft/client/gui/GuiGraphics;IIF)V # Used for menu interactions public net.minecraft.world.inventory.AbstractContainerMenu createCarriedSlotAccess()Lnet/minecraft/world/entity/SlotAccess; From fa3d3697621ff5d34ba7918bb9f3ea32eb324892 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sun, 10 Aug 2025 10:38:57 +0200 Subject: [PATCH 07/10] Fix more stuff --- .../runtime/actions/RecompileSourcesAction.java | 13 ++----------- .../actions/RecompileSourcesActionWithECJ.java | 6 ++++-- .../actions/RecompileSourcesActionWithJDK.java | 6 ++++-- .../runtime/actions/SelectSourcesToRecompile.java | 9 +++------ .../neoform/runtime/graph/ExecutionNode.java | 11 ----------- .../runtime/graph/transforms/ReplaceNodeOutput.java | 2 -- 6 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java index 08c337c5..9ed72846 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesAction.java @@ -1,6 +1,5 @@ package net.neoforged.neoform.runtime.actions; -import net.neoforged.neoform.runtime.artifacts.ClasspathItem; import net.neoforged.neoform.runtime.cache.CacheKeyBuilder; import net.neoforged.neoform.runtime.engine.ProcessingEnvironment; import net.neoforged.neoform.runtime.graph.ExecutionNodeAction; @@ -12,8 +11,8 @@ public abstract class RecompileSourcesAction extends BuiltInAction implements ExecutionNodeAction { - private ExtensibleClasspath classpath = new ExtensibleClasspath(); - private ExtensibleClasspath sourcepath = new ExtensibleClasspath(); + private final ExtensibleClasspath classpath = new ExtensibleClasspath(); + private final ExtensibleClasspath sourcepath = new ExtensibleClasspath(); private int targetJavaVersion = 21; @Override @@ -51,18 +50,10 @@ public ExtensibleClasspath getClasspath() { return classpath; } - public void setClasspath(ExtensibleClasspath classpath) { - this.classpath = classpath; - } - public ExtensibleClasspath getSourcepath() { return sourcepath; } - public void setSourcepath(ExtensibleClasspath sourcepath) { - this.sourcepath = sourcepath; - } - public int getTargetJavaVersion() { return targetJavaVersion; } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java index bd6bc572..eb833ab5 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithECJ.java @@ -215,8 +215,10 @@ public void computeCacheKey(CacheKeyBuilder ck) { @Override public RecompileSourcesAction copy() { var ret = new RecompileSourcesActionWithECJ(); - ret.setClasspath(getClasspath().copy()); - ret.setSourcepath(getSourcepath().copy()); + ret.getClasspath().setOverriddenClasspath(getClasspath().getOverriddenClasspath()); + ret.getClasspath().setAdditionalClasspath(getClasspath().getAdditionalClasspath()); + ret.getSourcepath().setOverriddenClasspath(getSourcepath().getOverriddenClasspath()); + ret.getSourcepath().setAdditionalClasspath(getSourcepath().getAdditionalClasspath()); ret.setTargetJavaVersion(getTargetJavaVersion()); return ret; } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java index 6ef8052f..35254410 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/RecompileSourcesActionWithJDK.java @@ -112,8 +112,10 @@ private List getCompilerOptions() { @Override public RecompileSourcesAction copy() { var ret = new RecompileSourcesActionWithJDK(); - ret.setClasspath(getClasspath().copy()); - ret.setSourcepath(getSourcepath().copy()); + ret.getClasspath().setOverriddenClasspath(getClasspath().getOverriddenClasspath()); + ret.getClasspath().setAdditionalClasspath(getClasspath().getAdditionalClasspath()); + ret.getSourcepath().setOverriddenClasspath(getSourcepath().getOverriddenClasspath()); + ret.getSourcepath().setAdditionalClasspath(getSourcepath().getAdditionalClasspath()); ret.setTargetJavaVersion(getTargetJavaVersion()); return ret; } diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java index e0bba8bb..8e80a7ce 100644 --- a/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java +++ b/src/main/java/net/neoforged/neoform/runtime/actions/SelectSourcesToRecompile.java @@ -89,13 +89,10 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt // Add trailing .java sourceName += ".java"; - if (!changedSourcePaths.contains(sourceName)) { - // Skip this entry - continue; + if (changedSourcePaths.contains(sourceName)) { + // Delete! + Files.delete(zfs.getPath(entry.getName())); } - - // Delete! - Files.delete(zfs.getPath(entry.getName())); } } } diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java b/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java index 462dd6bd..2af09fe3 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/ExecutionNode.java @@ -1,6 +1,5 @@ package net.neoforged.neoform.runtime.graph; -import net.neoforged.neoform.runtime.actions.ApplySourceTransformAction; import net.neoforged.neoform.runtime.utils.AnsiColor; import net.neoforged.neoform.runtime.utils.Logger; @@ -146,16 +145,6 @@ public ExecutionNodeAction action() { return action; } - // TODO: not sure yet if this is useful - public T actionAs(Class clazz) { - if (clazz.isInstance(action)) { - return (T) action; - } else { - throw new IllegalStateException("Node " + id + " has a different action type than expected. Expected: " - + clazz + " but got " + action.getClass()); - } - } - @Override public String toString() { return id; diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java index 7ce6c907..11046770 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java @@ -6,8 +6,6 @@ import net.neoforged.neoform.runtime.graph.ExecutionNodeBuilder; import net.neoforged.neoform.runtime.graph.NodeOutput; -import java.util.List; - public final class ReplaceNodeOutput extends GraphTransform { private final String nodeId; private final String outputId; From 1947e9ae5bc063870ac784b82564f49c56360eda Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sun, 10 Aug 2025 11:29:15 +0200 Subject: [PATCH 08/10] Also replace sources in sourcesWithNeoForge --- build.gradle | 2 +- .../neoform/runtime/cli/RunNeoFormCommand.java | 8 ++++++-- .../runtime/graph/transforms/ReplaceNodeOutput.java | 12 +++++++----- src/main/resources/tools.properties | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 8bff0255..e083ce76 100644 --- a/build.gradle +++ b/build.gradle @@ -181,7 +181,7 @@ idea { } "Experiments"(Application) { mainClass = mainClassName - programParameters = "run --verbose --partial-recompile --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --interface-injection-data=interfaces_test.json --validated-access-transformer=test_at.cfg --write-result=compiled:build/compiled-output.jar" + programParameters = "run --verbose --partial-recompile --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --interface-injection-data=interfaces_test.json --validated-access-transformer=test_at.cfg --write-result=node.sourcesWithNeoForge.output.output:build/sources-with-neoforge.jar" moduleRef(project, sourceSets.main) } "Run Neoform 1.21 (joined)"(Application) { diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index b04d42e3..c591c136 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -260,6 +260,7 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { String recompilationNodeName; if (partialRecompile) { + // TODO: probably not working as expected in 1.20.1 and older versions? // If there is a source transform (Parchment or NeoForge's AT), its output is the baseline. // Else it's the output of patch (NeoForm with no Parchment applied). var startingSources = graph.getNode("transformSources") != null @@ -274,12 +275,13 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { .make(applyAdditionalTransformsBuilder, startingSources); sourceTransformNode = applyAdditionalTransformsBuilder.build(); sourceTransformAction = (ApplySourceTransformAction) sourceTransformNode.action(); + var transformedSources = sourceTransformNode.getRequiredOutput("output"); // Split off sources that were modified by the second transform var selectSourcesBuilder = graph.nodeBuilder("selectSourcesToRecompile"); selectSourcesBuilder.input("originalSources", startingSources.asInput()); selectSourcesBuilder.input("originalClasses", recompileOutput.asInput()); - selectSourcesBuilder.input("transformedSources", sourceTransformNode.getRequiredOutput("output").asInput()); + selectSourcesBuilder.input("transformedSources", transformedSources.asInput()); var unchangedClasses = selectSourcesBuilder.output("unchangedClasses", NodeOutputType.JAR, "Classes that were already compiled and whose corresponding sources did not change."); var changedSourcesOnly = selectSourcesBuilder.output("changedSourcesOnly", NodeOutputType.JAR, "Sources that were changed and need to be recompiled."); selectSourcesBuilder.action(new SelectSourcesToRecompile()); @@ -305,7 +307,9 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { // Since we replace one node by many, we cannot use the ReplaceNodeOutput with the usual factory, // but rather directly call this helper - ReplaceNodeOutput.replaceOutput(graph, recompileOutput, injectedOutput, selectSources); + ReplaceNodeOutput.replaceOutput(graph, recompileOutput, injectedOutput, List.of(selectSources)); + // We also need to replace usages of the sources + ReplaceNodeOutput.replaceOutput(graph, startingSources, transformedSources, List.of(sourceTransformNode, selectSources)); recompilationNodeName = "recompileModifiedSources"; } else { diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java index 11046770..b40bdf04 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/transforms/ReplaceNodeOutput.java @@ -6,6 +6,8 @@ import net.neoforged.neoform.runtime.graph.ExecutionNodeBuilder; import net.neoforged.neoform.runtime.graph.NodeOutput; +import java.util.List; + public final class ReplaceNodeOutput extends GraphTransform { private final String nodeId; private final String outputId; @@ -45,18 +47,18 @@ public void apply(NeoFormEngine engine, ExecutionGraph graph) { var newNode = builder.build(); // Find all uses of the old output and replace them with our new output - replaceOutput(graph, originalOutput, newOutput, newNode); + replaceOutput(graph, originalOutput, newOutput, List.of(newNode)); } /** * General output -> output replacement. * - * @param toIgnore node for which the {@code originalOutput} should not be replaced by the {@code newOutput}, - * because it is part of the replacement pipeline + * @param toIgnore nodes for which the {@code originalOutput} should not be replaced by the {@code newOutput}, + * because they are part of the replacement pipeline */ - public static void replaceOutput(ExecutionGraph graph, NodeOutput originalOutput, NodeOutput newOutput, ExecutionNode toIgnore) { + public static void replaceOutput(ExecutionGraph graph, NodeOutput originalOutput, NodeOutput newOutput, List toIgnore) { for (var node : graph.getNodes()) { - if (node != toIgnore) { + if (!toIgnore.contains(node)) { for (var nodeInput : node.inputs().values()) { nodeInput.replaceReferences(originalOutput, newOutput); } diff --git a/src/main/resources/tools.properties b/src/main/resources/tools.properties index e8bbc6d0..55b29795 100644 --- a/src/main/resources/tools.properties +++ b/src/main/resources/tools.properties @@ -1,6 +1,6 @@ # https://projects.neoforged.net/neoforged/javasourcetransformer -JAVA_SOURCE_TRANSFORMER=net.neoforged.jst:jst-cli-bundle:1.0.74 +JAVA_SOURCE_TRANSFORMER=net.neoforged.jst:jst-cli-bundle:1.0.72 DIFF_PATCH=io.codechicken:DiffPatch:2.0.0.36:all From d5d4b37bcf92d39adfcbe3acccfee055bcbccfa4 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sun, 10 Aug 2025 11:33:11 +0200 Subject: [PATCH 09/10] Fix accidental replacement of recompile input --- .../net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index c591c136..0288496f 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -267,7 +267,8 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { ? graph.getRequiredOutput("transformSources", "output") : graph.getRequiredOutput("patch", "output"); var recompileOutput = graph.getRequiredOutput("recompile", "output"); - var recompileAction = (RecompileSourcesAction) recompileOutput.getNode().action(); + var recompileNode = recompileOutput.getNode(); + var recompileAction = (RecompileSourcesAction) recompileNode.action(); // Second transform action for additional transforms var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); @@ -309,7 +310,7 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { // but rather directly call this helper ReplaceNodeOutput.replaceOutput(graph, recompileOutput, injectedOutput, List.of(selectSources)); // We also need to replace usages of the sources - ReplaceNodeOutput.replaceOutput(graph, startingSources, transformedSources, List.of(sourceTransformNode, selectSources)); + ReplaceNodeOutput.replaceOutput(graph, startingSources, transformedSources, List.of(recompileNode, sourceTransformNode, selectSources)); recompilationNodeName = "recompileModifiedSources"; } else { From c0e21f2b1ce7be04eafddd777910ec7304fff7a4 Mon Sep 17 00:00:00 2001 From: Technici4n <13494793+Technici4n@users.noreply.github.com> Date: Sun, 10 Aug 2025 11:49:44 +0200 Subject: [PATCH 10/10] Get the sources from the recompile task's input --- .../neoforged/neoform/runtime/cli/RunNeoFormCommand.java | 7 +------ .../net/neoforged/neoform/runtime/graph/NodeInput.java | 7 +++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java index 0288496f..94304d58 100644 --- a/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java +++ b/src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java @@ -260,15 +260,10 @@ private void applyAdditionalTransforms(NeoFormEngine engine) { String recompilationNodeName; if (partialRecompile) { - // TODO: probably not working as expected in 1.20.1 and older versions? - // If there is a source transform (Parchment or NeoForge's AT), its output is the baseline. - // Else it's the output of patch (NeoForm with no Parchment applied). - var startingSources = graph.getNode("transformSources") != null - ? graph.getRequiredOutput("transformSources", "output") - : graph.getRequiredOutput("patch", "output"); var recompileOutput = graph.getRequiredOutput("recompile", "output"); var recompileNode = recompileOutput.getNode(); var recompileAction = (RecompileSourcesAction) recompileNode.action(); + var startingSources = recompileNode.getRequiredInput("sources").parentOutput(); // Second transform action for additional transforms var applyAdditionalTransformsBuilder = graph.nodeBuilder("applyAdditionalTransforms"); diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/NodeInput.java b/src/main/java/net/neoforged/neoform/runtime/graph/NodeInput.java index ac42ede4..4e096375 100644 --- a/src/main/java/net/neoforged/neoform/runtime/graph/NodeInput.java +++ b/src/main/java/net/neoforged/neoform/runtime/graph/NodeInput.java @@ -37,6 +37,8 @@ void setNode(ExecutionNode node) { public abstract T getValue(ResultRepresentation representation) throws IOException; + public abstract NodeOutput parentOutput(); + static final class NodeInputForOutput extends NodeInput { private NodeOutput output; @@ -66,5 +68,10 @@ public void collectCacheKeyComponent(CacheKeyBuilder builder) { public T getValue(ResultRepresentation representation) throws IOException { return output.getResultRepresentation(representation); } + + @Override + public NodeOutput parentOutput() { + return output; + } } }