Allow pack locators to load from the mods folder.

Regular mods are now automatically de-duped based on the version number -
the highest version file of a "root mod id" is now selected automatically.

Signed-off-by: cpw <cpw+github@weeksfamily.ca>

src/fmllauncher/java/net/minecraftforge/fml/loading/FMLEnvironment.java

@@ -20,9 +20,12 @@
 package net.minecraftforge.fml.loading;
 
 import cpw.mods.modlauncher.api.IEnvironment;
+import cpw.mods.modlauncher.api.TypesafeMap;
 import net.minecraftforge.api.distmarker.Dist;
 import net.minecraftforge.forgespi.Environment;
 
+import java.util.function.Supplier;
+
 public class FMLEnvironment
 {
     public static final Dist dist = FMLLoader.getDist();
@@ -32,4 +35,8 @@ public class FMLEnvironment
         environment.computePropertyIfAbsent(IEnvironment.Keys.NAMING.get(), v->naming);
         environment.computePropertyIfAbsent(Environment.Keys.DIST.get(), v->dist);
     }
+
+    public static class Keys {
+        public static final Supplier<TypesafeMap.Key<ClassLoader>> LOCATORCLASSLOADER = IEnvironment.buildKey("LOCATORCLASSLOADER",ClassLoader.class);
+    }
 }

src/fmllauncher/java/net/minecraftforge/fml/loading/LoadingModList.java

@@ -51,13 +51,18 @@ public class LoadingModList
 
     private LoadingModList(final List<ModFile> modFiles, final List<ModInfo> sortedList)
     {
-        this.modFiles = modFiles.stream().map(ModFile::getModFileInfo).map(ModFileInfo.class::cast).collect(Collectors.toList());
+        this.modFiles = modFiles.stream()
-        this.sortedList = sortedList.stream().
+                .map(ModFile::getModFileInfo)
-                map(ModInfo.class::cast).
+                .map(ModFileInfo.class::cast)
-                collect(Collectors.toList());
+                .collect(Collectors.toList());
-        this.fileById = this.modFiles.stream().map(ModFileInfo::getMods).flatMap(Collection::stream).
+        this.sortedList = sortedList.stream()
-                map(ModInfo.class::cast).
+                .map(ModInfo.class::cast)
-                collect(Collectors.toMap(ModInfo::getModId, ModInfo::getOwningFile));
+                .collect(Collectors.toList());
+        this.fileById = this.modFiles.stream()
+                .map(ModFileInfo::getMods)
+                .flatMap(Collection::stream)
+                .map(ModInfo.class::cast)
+                .collect(Collectors.toMap(ModInfo::getModId, ModInfo::getOwningFile));
         this.preLoadErrors = new ArrayList<>();
     }
 
@@ -76,19 +81,27 @@ public class LoadingModList
     }
     public void addCoreMods()
     {
-        modFiles.stream().map(ModFileInfo::getFile).map(ModFile::getCoreMods).flatMap(List::stream).forEach(FMLLoader.getCoreModProvider()::addCoreMod);
+        modFiles.stream()
+                .map(ModFileInfo::getFile)
+                .map(ModFile::getCoreMods)
+                .flatMap(List::stream)
+                .forEach(FMLLoader.getCoreModProvider()::addCoreMod);
     }
 
     public void addAccessTransformers()
     {
-        modFiles.stream().map(ModFileInfo::getFile).forEach(mod -> mod.getAccessTransformer().ifPresent(path -> FMLLoader.addAccessTransformer(path, mod)));
+        modFiles.stream()
+                .map(ModFileInfo::getFile)
+                .forEach(mod -> mod.getAccessTransformer().ifPresent(path -> FMLLoader.addAccessTransformer(path, mod)));
     }
 
     public void addForScanning(BackgroundScanHandler backgroundScanHandler)
     {
         this.scanner = backgroundScanHandler;
         backgroundScanHandler.setLoadingModList(this);
-        modFiles.stream().map(ModFileInfo::getFile).forEach(backgroundScanHandler::submitForScanning);
+        modFiles.stream()
+                .map(ModFileInfo::getFile)
+                .forEach(backgroundScanHandler::submitForScanning);
     }
 
     public List<ModFileInfo> getModFiles()

src/fmllauncher/java/net/minecraftforge/fml/loading/ModDirTransformerDiscoverer.java

@@ -28,36 +28,58 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.zip.ZipFile;
 
 public class ModDirTransformerDiscoverer implements ITransformerDiscoveryService {
     @Override
     public List<Path> candidates(final Path gameDirectory) {
+        ModDirTransformerDiscoverer.scan(gameDirectory);
+        return ModDirTransformerDiscoverer.transformers;
+    }
+
+    private static List<Path> transformers;
+
+    private static List<Path> locators;
+    public static List<Path> allExcluded() {
+        ArrayList<Path> paths = new ArrayList<>();
+        paths.addAll(transformers);
+        paths.addAll(locators);
+        return paths;
+    }
+
+    public static List<Path> getExtraLocators() {
+        return locators;
+    }
+
+    private static void scan(final Path gameDirectory) {
         final Path modsDir = gameDirectory.resolve(FMLPaths.MODSDIR.relative());
         if (!Files.exists(modsDir)) {
             // Skip if the mods dir doesn't exist yet.
-            return Collections.emptyList();
+            return;
         }
-        List<Path> paths = new ArrayList<>();
+        transformers = new ArrayList<>();
+        locators = new ArrayList<>();
         try {
             Files.createDirectories(modsDir);
-            Files.walk(modsDir, 1).forEach(p -> {
+            Files.walk(modsDir, 1).forEach(ModDirTransformerDiscoverer::visitFile);
-                if (!Files.isRegularFile(p)) return;
-                if (!p.toString().endsWith(".jar")) return;
-                if (LamdbaExceptionUtils.uncheck(()->Files.size(p)) == 0) return;
-                try (ZipFile zf = new ZipFile(new File(p.toUri()))) {
-                    if (zf.getEntry("META-INF/services/cpw.mods.modlauncher.api.ITransformationService") != null) {
-                        paths.add(p);
-                    }
-                } catch (IOException ioe) {
-                    LogManager.getLogger().error("Zip Error when loading jar file {}", p, ioe);
-                }
-            });
         } catch (IOException | IllegalStateException ioe) {
             LogManager.getLogger().error("Error during early discovery", ioe);
         }
-        return paths;
+    }
+
+    private static void visitFile(Path path) {
+        if (!Files.isRegularFile(path)) return;
+        if (!path.toString().endsWith(".jar")) return;
+        if (LamdbaExceptionUtils.uncheck(() -> Files.size(path)) == 0) return;
+        try (ZipFile zf = new ZipFile(new File(path.toUri()))) {
+            if (zf.getEntry("META-INF/services/cpw.mods.modlauncher.api.ITransformationService") != null) {
+                transformers.add(path.toRealPath());
+            } else if (zf.getEntry("META-INF/services/net.minecraftforge.forgespi.locating.IModLocator") != null) {
+                locators.add(path.toRealPath());
+            }
+        } catch (IOException ioe) {
+            LogManager.getLogger().error("Zip Error when loading jar file {}", path, ioe);
+        }
     }
 }

src/fmllauncher/java/net/minecraftforge/fml/loading/ModSorter.java

@@ -30,12 +30,14 @@ import net.minecraftforge.fml.loading.moddiscovery.ModFileInfo;
 import net.minecraftforge.fml.loading.moddiscovery.ModInfo;
 import net.minecraftforge.fml.loading.toposort.CyclePresentException;
 import net.minecraftforge.fml.loading.toposort.TopologicalSort;
+import net.minecraftforge.forgespi.locating.IModFile;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.util.StringBuilderFormattable;
 import org.apache.maven.artifact.versioning.ArtifactVersion;
 import org.apache.maven.artifact.versioning.DefaultArtifactVersion;
 
+import java.util.AbstractMap;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -45,7 +47,6 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static net.minecraftforge.fml.loading.LogMarkers.LOADING;
 
@@ -87,11 +88,19 @@ public class ModSorter
         // lambdas are identity based, so sorting them is impossible unless you hold reference to them
         final MutableGraph<ModFileInfo> graph = GraphBuilder.directed().build();
         AtomicInteger counter = new AtomicInteger();
-        Map<IModFileInfo, Integer> infos = modFiles.stream().map(ModFile::getModFileInfo).collect(Collectors.toMap(Function.identity(), (e) -> counter.incrementAndGet()));
+        Map<IModFileInfo, Integer> infos = modFiles
+                .stream()
+                .map(ModFile::getModFileInfo)
+                .collect(Collectors.toMap(Function.identity(), e -> counter.incrementAndGet()));
         infos.keySet().forEach(i -> graph.addNode((ModFileInfo) i));
-        modFiles.stream().map(ModFile::getModInfos).flatMap(Collection::stream).
+        modFiles
-                map(IModInfo::getDependencies).flatMap(Collection::stream).
+                .stream()
-                forEach(dep -> addDependency(graph, dep));
+                .map(ModFile::getModInfos)
+                .flatMap(Collection::stream)
+                .map(IModInfo::getDependencies)
+                .flatMap(Collection::stream)
+                .forEach(dep -> addDependency(graph, dep));
+
         final List<ModFileInfo> sorted;
         try
         {
@@ -109,15 +118,24 @@ public class ModSorter
             List<ExceptionData> dataList = cycles.stream()
                     .map(Set::stream)
                     .map(stream -> stream
-                            .flatMap(modFileInfo -> modFileInfo.getMods().stream()
+                            .flatMap(modFileInfo -> modFileInfo.getMods()
-                                    .map(IModInfo::getModId)).collect(Collectors.toList()))
+                                    .stream()
+                                    .map(IModInfo::getModId))
+                            .collect(Collectors.toList()))
                     .map(list -> new ExceptionData("fml.modloading.cycle", list))
                     .collect(Collectors.toList());
             throw new EarlyLoadingException("Sorting error", e, dataList);
         }
-        this.sortedList = sorted.stream().map(ModFileInfo::getMods).
+        this.sortedList = sorted
-                flatMap(Collection::stream).map(ModInfo.class::cast).collect(Collectors.toList());
+                .stream()
-        this.modFiles = sorted.stream().map(ModFileInfo::getFile).collect(Collectors.toList());
+                .map(ModFileInfo::getMods)
+                .flatMap(Collection::stream)
+                .map(ModInfo.class::cast)
+                .collect(Collectors.toList());
+        this.modFiles = sorted
+                .stream()
+                .map(ModFileInfo::getFile)
+                .collect(Collectors.toList());
     }
 
     @SuppressWarnings("UnstableApiUsage")
@@ -144,13 +162,27 @@ public class ModSorter
 
     private void buildUniqueList()
     {
-        final Stream<ModInfo> modInfos = modFiles.stream()
+        // Collect mod files by first modid in the file. This will be used for deduping purposes
+        final Map<String, List<IModFile>> modFilesByFirstId = modFiles
+                .stream()
+                .collect(Collectors.groupingBy(mf -> mf.getModInfos().get(0).getModId()));
+
+        // Select the newest by artifact version sorting of non-unique files thus identified
+        this.modFiles = modFilesByFirstId.entrySet().stream()
+                .map(this::selectNewestModInfo)
+                .map(Map.Entry::getValue)
+                .map(ModFile.class::cast)
+                .collect(Collectors.toList());
+
+        // Transform to the full mod id list
+        final Map<String, List<ModInfo>> modIds = modFiles.stream()
                 .map(ModFile::getModInfos)
                 .flatMap(Collection::stream)
-                .map(ModInfo.class::cast);
+                .map(ModInfo.class::cast)
-        final Map<String, List<ModInfo>> modIds = modInfos.collect(Collectors.groupingBy(IModInfo::getModId));
+                .collect(Collectors.groupingBy(IModInfo::getModId));
 
-        // TODO: make this figure out dupe handling better
+        // Its theoretically possible that some mod has somehow moved an id to a secondary place, thus causing a dupe.
+        // We can't handle this
         final List<Map.Entry<String, List<ModInfo>>> dupedMods = modIds
                 .entrySet()
                 .stream()
@@ -171,30 +203,58 @@ public class ModSorter
                 .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get(0)));
     }
 
+    private Map.Entry<String, IModFile> selectNewestModInfo(Map.Entry<String, List<IModFile>> fullList) {
+        List<IModFile> modInfoList = fullList.getValue();
+        if (modInfoList.size() > 1) {
+            LOGGER.debug("Found {} mods for first modid {}, selecting most recent based on version data", modInfoList.size(), fullList.getKey());
+            modInfoList.sort(Comparator.<IModFile, ArtifactVersion>comparing(mf -> mf.getModInfos().get(0).getVersion()).reversed());
+            LOGGER.debug("Selected file {} for modid {} with version {}", modInfoList.get(0).getFileName(), fullList.getKey(), modInfoList.get(0).getModInfos().get(0).getVersion());
+        }
+        return new AbstractMap.SimpleImmutableEntry<>(fullList.getKey(), modInfoList.get(0));
+    }
+
     private void verifyDependencyVersions()
     {
-        final Map<String, ArtifactVersion> modVersions = modFiles.stream().map(ModFile::getModInfos).
+        final Map<String, ArtifactVersion> modVersions = modFiles
-                flatMap(Collection::stream).collect(Collectors.toMap(IModInfo::getModId, IModInfo::getVersion));
+                .stream()
-        final Map<IModInfo, List<IModInfo.ModVersion>> modVersionDependencies = modFiles.stream().
+                .map(ModFile::getModInfos)
-                map(ModFile::getModInfos).flatMap(Collection::stream).
+                .flatMap(Collection::stream)
-                collect(Collectors.groupingBy(Function.identity(), Java9BackportUtils.flatMapping(e -> e.getDependencies().stream(), Collectors.toList())));
+                .collect(Collectors.toMap(IModInfo::getModId, IModInfo::getVersion));
-        final Set<IModInfo.ModVersion> mandatoryModVersions = modVersionDependencies.values().stream().flatMap(Collection::stream).
+
-                filter(mv -> mv.isMandatory() && mv.getSide().isCorrectSide()).collect(Collectors.toSet());
+        final Map<IModInfo, List<IModInfo.ModVersion>> modVersionDependencies = modFiles
+                .stream()
+                .map(ModFile::getModInfos)
+                .flatMap(Collection::stream)
+                .collect(Collectors.groupingBy(Function.identity(), Java9BackportUtils.flatMapping(e -> e.getDependencies().stream(), Collectors.toList())));
+
+        final Set<IModInfo.ModVersion> mandatoryModVersions = modVersionDependencies
+                .values()
+                .stream()
+                .flatMap(Collection::stream)
+                .filter(mv -> mv.isMandatory() && mv.getSide().isCorrectSide())
+                .collect(Collectors.toSet());
+
         LOGGER.debug(LOADING, "Found {} mandatory requirements", mandatoryModVersions.size());
-        final Set<IModInfo.ModVersion> missingVersions = mandatoryModVersions.stream().filter(mv->this.modVersionMatches(mv, modVersions)).collect(Collectors.toSet());
+        final Set<IModInfo.ModVersion> missingVersions = mandatoryModVersions
+                .stream()
+                .filter(mv->this.modVersionMatches(mv, modVersions))
+                .collect(Collectors.toSet());
         LOGGER.debug(LOADING, "Found {} mandatory mod requirements missing", missingVersions.size());
 
         if (!missingVersions.isEmpty()) {
-            final List<EarlyLoadingException.ExceptionData> exceptionData = missingVersions.stream().map(mv ->
+            final List<EarlyLoadingException.ExceptionData> exceptionData = missingVersions
-                    new EarlyLoadingException.ExceptionData("fml.modloading.missingdependency", mv.getModId(),
+                    .stream()
-                            mv.getOwner().getModId(), mv.getVersionRange(), modVersions.getOrDefault(mv.getModId(), new DefaultArtifactVersion("null")))).
+                    .map(mv -> new EarlyLoadingException.ExceptionData("fml.modloading.missingdependency",
-                    collect(Collectors.toList());
+                            mv.getModId(), mv.getOwner().getModId(), mv.getVersionRange(),
+                            modVersions.getOrDefault(mv.getModId(), new DefaultArtifactVersion("null"))))
+                    .collect(Collectors.toList());
             throw new EarlyLoadingException("Missing mods", null, exceptionData);
         }
     }
 
     private boolean modVersionMatches(final IModInfo.ModVersion mv, final Map<String, ArtifactVersion> modVersions)
     {
-        return !modVersions.containsKey(mv.getModId()) || !mv.getVersionRange().containsVersion(modVersions.get(mv.getModId()));
+        return !(modVersions.containsKey(mv.getModId()) &&
+                (mv.getVersionRange().containsVersion(modVersions.get(mv.getModId())) || modVersions.get(mv.getModId()).toString().equals("NONE")));
     }
 }

src/fmllauncher/java/net/minecraftforge/fml/loading/moddiscovery/ModDiscoverer.java

@@ -19,10 +19,14 @@
 
 package net.minecraftforge.fml.loading.moddiscovery;
 
+import cpw.mods.gross.Java9ClassLoaderUtil;
 import cpw.mods.modlauncher.Launcher;
 import cpw.mods.modlauncher.ServiceLoaderStreamUtils;
+import cpw.mods.modlauncher.api.LamdbaExceptionUtils;
+import net.minecraftforge.fml.loading.FMLEnvironment;
 import net.minecraftforge.fml.loading.FMLLoader;
 import net.minecraftforge.fml.loading.LoadingModList;
+import net.minecraftforge.fml.loading.ModDirTransformerDiscoverer;
 import net.minecraftforge.fml.loading.ModSorter;
 import net.minecraftforge.fml.loading.progress.StartupMessageManager;
 import net.minecraftforge.forgespi.Environment;
@@ -34,6 +38,8 @@ import org.apache.logging.log4j.Logger;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
@@ -64,11 +70,18 @@ public class ModDiscoverer {
     private static final Logger LOGGER = LogManager.getLogger();
     private final ServiceLoader<IModLocator> locators;
     private final List<IModLocator> locatorList;
+    private final LocatorClassLoader locatorClassLoader;
 
     public ModDiscoverer(Map<String, ?> arguments) {
         Launcher.INSTANCE.environment().computePropertyIfAbsent(Environment.Keys.MODFOLDERFACTORY.get(), v->ModsFolderLocator::new);
         Launcher.INSTANCE.environment().computePropertyIfAbsent(Environment.Keys.PROGRESSMESSAGE.get(), v->StartupMessageManager.locatorConsumer().orElseGet(()->s->{}));
-        locators = ServiceLoader.load(IModLocator.class);
+        locatorClassLoader = new LocatorClassLoader();
+        Launcher.INSTANCE.environment().computePropertyIfAbsent(FMLEnvironment.Keys.LOCATORCLASSLOADER.get(), v->locatorClassLoader);
+        ModDirTransformerDiscoverer.getExtraLocators()
+                .stream()
+                .map(LamdbaExceptionUtils.rethrowFunction(p->p.toUri().toURL()))
+                .forEach(locatorClassLoader::addURL);
+        locators = ServiceLoader.load(IModLocator.class, locatorClassLoader);
         locatorList = ServiceLoaderStreamUtils.toList(this.locators);
         locatorList.forEach(l->l.initArguments(arguments));
         locatorList.add(new MinecraftLocator());
@@ -77,6 +90,7 @@ public class ModDiscoverer {
 
     ModDiscoverer(List<IModLocator> locatorList) {
         this.locatorList = locatorList;
+        this.locatorClassLoader = null;
         this.locators = null;
     }
 
@@ -114,6 +128,16 @@ public class ModDiscoverer {
         return backgroundScanHandler;
     }
 
+    private static class LocatorClassLoader extends URLClassLoader {
+        LocatorClassLoader() {
+            super(Java9ClassLoaderUtil.getSystemClassPathURLs(), getSystemClassLoader());
+        }
+
+        @Override
+        protected void addURL(final URL url) {
+            super.addURL(url);
+        }
+    }
     private static class MinecraftLocator implements IModLocator {
         private final Path mcJar = FMLLoader.getMCPaths()[0];
         private final FileSystem fileSystem;

src/fmllauncher/java/net/minecraftforge/fml/loading/moddiscovery/ModsFolderLocator.java

@@ -56,14 +56,14 @@ public class ModsFolderLocator extends AbstractJarFileLocator {
     @Override
     public List<IModFile> scanMods() {
         LOGGER.debug(SCAN,"Scanning mods dir {} for mods", this.modFolder);
-        List<Path> excluded = new ModDirTransformerDiscoverer().candidates(FMLPaths.GAMEDIR.get());
+        List<Path> excluded = ModDirTransformerDiscoverer.allExcluded();
-        return uncheck(()-> Files.list(this.modFolder)).
+        return uncheck(()-> Files.list(this.modFolder))
-                filter(p->!excluded.contains(p)).
+                .filter(p->!excluded.contains(p))
-                sorted(Comparator.comparing(path-> StringUtils.toLowerCase(path.getFileName().toString()))).
+                .sorted(Comparator.comparing(path-> StringUtils.toLowerCase(path.getFileName().toString())))
-                filter(p->StringUtils.toLowerCase(p.getFileName().toString()).endsWith(SUFFIX)).
+                .filter(p->StringUtils.toLowerCase(p.getFileName().toString()).endsWith(SUFFIX))
-                map(p->new ModFile(p, this)).
+                .map(p->new ModFile(p, this))
-                peek(f->modJars.compute(f, (mf, fs)->createFileSystem(mf))).
+                .peek(f->modJars.compute(f, (mf, fs)->createFileSystem(mf)))
-                collect(Collectors.toList());
+                .collect(Collectors.toList());
     }
 
     @Override