[BUG] [JSR-199] ECJ cannot resolve JPMS modules if using a user-provi…

…ded file manager (#3446)

ClasspathJsr199:
+ change to 1 ClasspathJsr199 per module
+ various additions for handling modules
+ capability to handle sources
+ implement listPackages() using JavaFileObjects
ModuleFinder:
+ find automatic modules, too
+ allow invocation without a parser for binary-only scans
EclipseFileManager:
+ allow scanning all packages (by specifying an empty package name)
+ establish symmetry between setLocation() and setLocationFromPaths()
+ catch disruptive IAE from ModuleFinder.scanForModule()

More fixes:
+ encoding: pass through from Main.compilerOptions
+ consistently extract module and pass it into CUs and class readers
+ implement module extraction from JavaFileObject (src & class)
+ detect initial / duplicate source files by JavaFileObject, not path
+ fix arg passing in ProblemReporter.invalidPackageReference()

Tests:
+ implement package matching in InMemoryJavaFileManager.list()
+ test with two module dependencies (one auto, one regular)

Fixes #958

org.eclipse.jdt.compiler.tool.tests/resources/module_locations/GH958/module-info.java

@@ -0,0 +1,3 @@
+module org.example {
+    requires org.example.adder;
+}

org.eclipse.jdt.compiler.tool.tests/resources/module_locations/GH958/org/example/impl/AddNumbers.java

@@ -0,0 +1,11 @@
+package org.example.impl;
+
+import org.example.adder.Adder;
+
+public class AddNumbers {
+    public static void main(String[] args) {
+        int a = 123;
+        int b = 456;
+        System.out.println(new Adder().add(a, b));
+    }
+}

org.eclipse.jdt.compiler.tool.tests/resources/module_locations/GH958_2/module-info.java

@@ -0,0 +1,4 @@
+module GH958_2 {
+    requires GH958_mod;
+    requires org.example.adder;
+}

org.eclipse.jdt.compiler.tool.tests/resources/module_locations/GH958_2/org/example2/Main.java

@@ -0,0 +1,13 @@
+package org.example2;
+
+import org.example.regular.Foo;
+import org.example.adder.Adder;
+
+public class Main {
+	Foo foo;
+    public static void main(String[] args) {
+        int a = 123;
+        int b = 456;
+        System.out.println(new Adder().add(a, b));
+    }
+}

org.eclipse.jdt.compiler.tool.tests/src/org/eclipse/jdt/compiler/tool/tests/CompilerToolJava9Tests.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2017, 2021 IBM Corporation and others.
+ * Copyright (c) 2017, 2024 IBM Corporation and others.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -21,6 +21,9 @@
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
 import java.net.URL;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -954,6 +957,93 @@ public void testBug533830_1() throws IOException {
 		//-- We must now also have a diagnostic
 		assertTrue("The diagnostic listener did not receive an error for the illegal option", b.listener().hasDiagnostic("option -source is not supported when --release is used"));
 	}
+
+	/**
+	 * Proxy that transparently delegates to a JavaFileManager impl, but hides the physical
+	 * implementation of the delegated file manager to prevent ECJ performing analysis based
+	 * on the superclasses of the file manager object.
+	 */
+	static class DemotingFileManagerProxy implements InvocationHandler {
+		private final JavaFileManager fileManager;
+
+		private DemotingFileManagerProxy(JavaFileManager fileManager) {
+			this.fileManager = fileManager;
+		}
+
+		@Override
+		public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+			System.err.printf("Calling %s(%s)%n", method.getName(), Arrays.toString(args));
+			try {
+				var result = method.invoke(fileManager, args);
+				System.err.printf("Call to %s(%s) returned %s%n", method.getName(), Arrays.toString(args), result);
+				return result;
+			} catch (Throwable ex) {
+				ex.printStackTrace();
+				throw ex;
+			}
+		}
+
+		public static JavaFileManager demote(JavaFileManager fileManager) {
+			return (JavaFileManager) Proxy.newProxyInstance(
+					fileManager.getClass().getClassLoader(),
+					new Class<?>[]{ JavaFileManager.class },
+					new DemotingFileManagerProxy(fileManager)
+					);
+		}
+	}
+	public void testGH958() throws Exception {
+		File classOutput = new File(_tmpFolder);
+		JavaCompiler compiler = new EclipseCompiler();
+		StandardJavaFileManager standardFileManager = compiler.getStandardFileManager(null, null, null);
+
+		List<File> modulePath = List.of(new File("resources/module_locations/GH958_automod.jar"));
+		List<File> sourcePath = List.of(new File("resources/module_locations/GH958"));
+		standardFileManager.setLocation(StandardLocation.MODULE_PATH, modulePath);
+		standardFileManager.setLocation(StandardLocation.SOURCE_PATH, sourcePath);
+		standardFileManager.setLocation(StandardLocation.CLASS_OUTPUT, List.of(classOutput));
+
+		// Make ECJ think we don't inherit StandardJavaFileManager by wrapping it.
+		JavaFileManager demotedFileManager = DemotingFileManagerProxy.demote(standardFileManager);
+		Iterable<JavaFileObject> compilationUnits = demotedFileManager.list(StandardLocation.SOURCE_PATH, "", Set.of(JavaFileObject.Kind.SOURCE), true);
+
+		CompilationTask task = compiler.getTask(
+				null,
+				demotedFileManager,
+				null,
+				List.of("--release", "11", "-verbose"),
+				List.of(),
+				compilationUnits
+		);
+
+		assertTrue(task.call());
+	}
+	public void testGH958_2modules() throws Exception {
+		File classOutput = new File(_tmpFolder);
+		JavaCompiler compiler = new EclipseCompiler();
+		StandardJavaFileManager standardFileManager = compiler.getStandardFileManager(null, null, null);
+
+		List<File> modulesPath = List.of(new File("resources/module_locations/GH958_automod.jar"),
+											new File("resources/module_locations/GH958_mod.jar"));
+		List<File> sourcePath = List.of(new File("resources/module_locations/GH958_2"));
+		standardFileManager.setLocation(StandardLocation.MODULE_PATH, modulesPath);
+		standardFileManager.setLocation(StandardLocation.SOURCE_PATH, sourcePath);
+		standardFileManager.setLocation(StandardLocation.CLASS_OUTPUT, List.of(classOutput));
+
+		// Make ECJ think we don't inherit StandardJavaFileManager by wrapping it.
+		JavaFileManager demotedFileManager = DemotingFileManagerProxy.demote(standardFileManager);
+		Iterable<JavaFileObject> compilationUnits = demotedFileManager.list(StandardLocation.SOURCE_PATH, "", Set.of(JavaFileObject.Kind.SOURCE), true);
+
+		CompilationTask task = compiler.getTask(
+				null,
+				demotedFileManager,
+				null,
+				List.of("--release", "11", "-verbose"),
+				List.of(),
+				compilationUnits
+		);
+
+		assertTrue(task.call());
+	}
 
 	/**
 	 * Helps with building a compiler invocation, handling the common parts of testing.

org.eclipse.jdt.compiler.tool.tests/src/org/eclipse/jdt/compiler/tool/tests/InMemoryCompilationTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2021 IBM Corporation and others.
+ * Copyright (c) 2021, 2024 IBM Corporation and others.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -191,7 +191,16 @@ public Iterable<JavaFileObject> list(Location location, String packageName, Set<
 				throws IOException {
 			List<JavaFileObject> result = new ArrayList<>();
 			if (location == StandardLocation.SOURCE_PATH && kinds.contains(Kind.SOURCE)) {
-				result.addAll(sources);
+				for (InMemoryJavaSourceFileObject sourceFileObject : sources) {
+					String name = sourceFileObject.getAbsClassName();
+					int lastDot = name.lastIndexOf('.');
+					if (lastDot == -1)
+						continue;
+					String packName = name.substring(0, lastDot-1);
+					boolean match = recurse ? packName.startsWith(packageName) : packName.equals(packageName);
+					if (match)
+						result.add(sourceFileObject);
+				}
 			}
 			if (super.hasLocation(location)) {
 				Iterable<JavaFileObject> superResult = super.list(location, packageName, kinds, recurse);

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/apt/dispatch/BatchFilerImpl.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2018 BEA Systems, Inc.
+ * Copyright (c) 2006, 2024 BEA Systems, Inc.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -19,6 +19,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.util.Collections;
 import java.util.HashSet;
 import javax.annotation.processing.Filer;
 import javax.annotation.processing.FilerException;
@@ -28,8 +29,12 @@
 import javax.tools.JavaFileManager;
 import javax.tools.JavaFileManager.Location;
 import javax.tools.JavaFileObject;
+import javax.tools.JavaFileObject.Kind;
 import javax.tools.StandardLocation;
+import org.eclipse.jdt.internal.compiler.batch.ClasspathJsr199;
+import org.eclipse.jdt.internal.compiler.batch.Main;
 import org.eclipse.jdt.internal.compiler.env.ICompilationUnit;
+import org.eclipse.jdt.internal.compiler.env.IModule;
 import org.eclipse.jdt.internal.compiler.lookup.ReferenceBinding;
 
 /**
@@ -43,13 +48,32 @@ public class BatchFilerImpl implements Filer {
 	protected final BatchProcessingEnvImpl _env;
 	protected final JavaFileManager _fileManager;
 	protected final HashSet<URI> _createdFiles;
+	protected String _moduleName;
+	protected String _encoding;
 
-	public BatchFilerImpl(BaseAnnotationProcessorManager dispatchManager, BatchProcessingEnvImpl env)
+	public BatchFilerImpl(BaseAnnotationProcessorManager dispatchManager, BatchProcessingEnvImpl env, Main main)
 	{
 		this._dispatchManager = dispatchManager;
 		this._fileManager = env._fileManager;
 		this._env = env;
 		this._createdFiles = new HashSet<>();
+		this._encoding = main.getDefaultEncoding();
+		if (this._fileManager.hasLocation(StandardLocation.SOURCE_PATH)) {
+			try {
+				for (JavaFileObject javaFileObject : this._fileManager.list(StandardLocation.SOURCE_PATH, "", //$NON-NLS-1$
+						Collections.singleton(Kind.SOURCE), false)) {
+					if (javaFileObject.getName().equals(IModule.MODULE_INFO_JAVA)) {
+						IModule module = ClasspathJsr199.extractModuleFromFileObject(javaFileObject, main::getNewParser, null, this._encoding);
+						if (module != null)
+							this._moduleName = String.valueOf(module.name());
+						break;
+					}
+				}
+			} catch (IOException e) {
+				e.printStackTrace();
+				main.logger.logException(e);
+			}
+		}
 	}
 
 	public void addNewUnit(ICompilationUnit unit) {
@@ -74,7 +98,7 @@ public JavaFileObject createClassFile(CharSequence name,
 		}
 
 		this._createdFiles.add(uri);
-		return new HookedJavaFileObject(jfo, jfo.getName(), name.toString(), this);
+		return new HookedJavaFileObject(jfo, jfo.getName(), name.toString(), this, this._moduleName, this._encoding);
 	}
 
 	/* (non-Javadoc)
@@ -146,7 +170,13 @@ public JavaFileObject createSourceFile(CharSequence name,
 		if (typeElement != null) {
 			throw new FilerException("Source file already exists : " + moduleAndPkgString); //$NON-NLS-1$
 		}
-		Location location = mod == null ? StandardLocation.SOURCE_OUTPUT : this._fileManager.getLocationForModule(StandardLocation.SOURCE_OUTPUT, mod);
+		Location location;
+		if (mod == null) {
+			location = StandardLocation.SOURCE_OUTPUT;
+			mod = this._moduleName;
+		} else {
+			location = this._fileManager.getLocationForModule(StandardLocation.SOURCE_OUTPUT, mod);
+		}
 		JavaFileObject jfo = this._fileManager.getJavaFileForOutput(location, name.toString(), JavaFileObject.Kind.SOURCE, null);
 		URI uri = jfo.toUri();
 		if (this._createdFiles.contains(uri)) {
@@ -155,7 +185,7 @@ public JavaFileObject createSourceFile(CharSequence name,
 
 		this._createdFiles.add(uri);
 		// hook the file object's writers to create compilation unit and add to addedUnits()
-		return new HookedJavaFileObject(jfo, jfo.getName(), name.toString(), this);
+		return new HookedJavaFileObject(jfo, jfo.getName(), name.toString(), this, mod, this._encoding);
 	}
 
 	/* (non-Javadoc)

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/apt/dispatch/BatchProcessingEnvImpl.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2023 BEA Systems, Inc.
+ * Copyright (c) 2006, 2024 BEA Systems, Inc.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -89,7 +89,7 @@ public BatchProcessingEnvImpl(BaseAnnotationProcessorManager dispatchManager, Ma
 			this._fileManager = manager;
 		}
 		this._processorOptions = Collections.unmodifiableMap(parseProcessorOptions(commandLineArguments));
-		this._filer = new BatchFilerImpl(this._dispatchManager, this);
+		this._filer = new BatchFilerImpl(this._dispatchManager, this, this._compilerOwner);
 		this._messager = new BatchMessagerImpl(this, this._compilerOwner);
 	}
 

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/apt/dispatch/HookedJavaFileObject.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2006, 2023 BEA Systems, Inc. and others
+ * Copyright (c) 2006, 2024 BEA Systems, Inc. and others
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -27,6 +27,7 @@
 import org.eclipse.jdt.internal.compiler.env.IBinaryType;
 import org.eclipse.jdt.internal.compiler.lookup.BinaryTypeBinding;
 import org.eclipse.jdt.internal.compiler.lookup.ReferenceBinding;
+import org.eclipse.jdt.internal.compiler.problem.AbortCompilationUnit;
 
 /**
  * A delegating JavaFileObject that hooks the close() methods of the Writer
@@ -200,11 +201,17 @@ public String toString() {
 
 	private final String _typeName;
 
-	public HookedJavaFileObject(JavaFileObject fileObject, String fileName, String typeName, BatchFilerImpl filer) {
+	private final String _moduleName;
+
+	private final String _encoding;
+
+	public HookedJavaFileObject(JavaFileObject fileObject, String fileName, String typeName, BatchFilerImpl filer, String module, String encoding) {
 		super(fileObject);
 		this._filer = filer;
 		this._fileName = fileName;
 		this._typeName = typeName;
+		this._moduleName = module;
+		this._encoding = encoding;
 	}
 
 	@SuppressWarnings("resource") // ForwardingOutputStream forwards close() too
@@ -222,11 +229,21 @@ public Writer openWriter() throws IOException {
 	protected void closed() {
 		if (!this._closed) {
 			this._closed = true;
-			//TODO: support encoding
 			switch(this.getKind()) {
 				case SOURCE :
-					CompilationUnit unit = new CompilationUnit(null, this._fileName, null /* encoding */, null, this._filer._env.shouldIgnoreOptionalProblems(this._fileName.toCharArray()), null);
-					this._filer.addNewUnit(unit);
+					try {
+						CompilationUnit unit = new CompilationUnit(
+								getCharContent(false).toString().toCharArray(),
+								this._fileName,
+								this._encoding,
+								null /* destination path */,
+								this._filer._env.shouldIgnoreOptionalProblems(this._fileName.toCharArray()),
+								this._moduleName);
+						this._filer.addNewUnit(unit);
+					} catch (IOException e) {
+						e.printStackTrace();
+						throw new AbortCompilationUnit(null, e, this._encoding);
+					}
 					break;
 				case CLASS :
 					IBinaryType binaryType = null;