The testrunner now uses a different mechanism to verify correctness of produced errors and warnings (i.e. we intentionally compile code with errors in them to verify that the appropriate error or warning message is emitted when lombok is active during a compilation run of either javac or ecj) - instead of string comparisons, it's a little more complex. This to enable testing of both javac6 and javac7, even if they produce (slightly) different error output.

Updated all message files in the 'expected output' directories to represent this change.

5 files changed, 177 insertions, 10 deletions

diff --git a/test/core/src/lombok/AbstractRunTests.java b/test/core/src/lombok/AbstractRunTests.java
index 02c4803a..676426f4 100644
--- a/test/core/src/lombok/AbstractRunTests.java
+++ b/test/core/src/lombok/AbstractRunTests.java
@@ -25,13 +25,19 @@ import static org.junit.Assert.*;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 
 public abstract class AbstractRunTests {
@@ -43,11 +49,23 @@ public abstract class AbstractRunTests {
 	}
 	
 	public boolean compareFile(DirectoryRunner.TestParams params, File file) throws Throwable {
-		StringBuilder messages = new StringBuilder();
+		LinkedHashSet<CompilerMessage> messages = new LinkedHashSet<CompilerMessage>();
 		StringWriter writer = new StringWriter();
 		transformCode(messages, writer, file);
 		String expectedFile = readFile(params.getAfterDirectory(), file, false);
-		String expectedMessages = readFile(params.getMessagesDirectory(), file, true);
+		List<CompilerMessageMatcher> expectedMessages = Collections.emptyList();
+		if (params.getMessagesDirectory() != null) {
+			try {
+				InputStream in = new FileInputStream(new File(params.getMessagesDirectory(), file.getName() + ".messages"));
+				try {
+					expectedMessages = CompilerMessageMatcher.readAll(in);
+				} finally {
+					in.close();
+				}
+			} catch (FileNotFoundException ex) {
+				// That's okay - then we expect no messages, and expectedMessages already gets initialized to the empty list.
+			}
+		}
 		
 		StringReader r = new StringReader(expectedFile);
 		BufferedReader br = new BufferedReader(r);
@@ -58,13 +76,13 @@ public abstract class AbstractRunTests {
 				expectedFile,
 				writer.toString(),
 				expectedMessages,
-				messages.toString(),
+				messages,
 				params.printErrors());
 		
 		return true;
 	}
 	
-	protected abstract void transformCode(StringBuilder message, StringWriter result, File file) throws Throwable;
+	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable;
 	
 	protected String readFile(File file) throws IOException {
 		BufferedReader reader;
@@ -107,7 +125,19 @@ public abstract class AbstractRunTests {
 		}
 	}
 	
-	private void compare(String name, String expectedFile, String actualFile, String expectedMessages, String actualMessages, boolean printErrors) throws Throwable {
+	private static void dumpToFile(File file, Collection<CompilerMessage> content) throws IOException {
+		FileOutputStream fos = new FileOutputStream(file);
+		try {
+			for (CompilerMessage message : content) {
+				fos.write(message.asCompilerMessageMatcher().toString().getBytes("UTF-8"));
+				fos.write('\n');
+			}
+		} finally {
+			fos.close();
+		}
+	}
+	
+	private void compare(String name, String expectedFile, String actualFile, List<CompilerMessageMatcher> expectedMessages, LinkedHashSet<CompilerMessage> actualMessages, boolean printErrors) throws Throwable {
 		try {
 			compareContent(name, expectedFile, actualFile);
 		} catch (Throwable e) {
@@ -129,8 +159,9 @@ public abstract class AbstractRunTests {
 			}
 			throw e;
 		}
+		
 		try {
-			compareContent(name, expectedMessages, actualMessages);
+			compareMessages(name, expectedMessages, actualMessages);
 		} catch (Throwable e) {
 			if (printErrors) {
 				System.out.println("***** " + name + " *****");
@@ -148,6 +179,27 @@ public abstract class AbstractRunTests {
 		}
 	}
 	
+	private static void compareMessages(String name, List<CompilerMessageMatcher> expected, LinkedHashSet<CompilerMessage> actual) {
+		Iterator<CompilerMessageMatcher> expectedIterator = expected.iterator();
+		Iterator<CompilerMessage> actualIterator = actual.iterator();
+		
+		while (true) {
+			boolean exHasNext = expectedIterator.hasNext();
+			boolean acHasNext = actualIterator.hasNext();
+			if (!exHasNext && !acHasNext) break;
+			if (exHasNext && acHasNext) {
+				CompilerMessageMatcher cmm = expectedIterator.next();
+				CompilerMessage cm = actualIterator.next();
+				if (cmm.matches(cm)) continue;
+				fail(String.format("[%s] Expected message '%s' but got message '%s'", name, cmm, cm));
+				throw new AssertionError("fail should have aborted already.");
+			}
+			if (exHasNext) fail(String.format("[%s] Expected message '%s' but ran out of actual messages", name, expectedIterator.next()));
+			if (acHasNext) fail(String.format("[%s] Unexpected message: %s", name, actualIterator.next()));
+			throw new AssertionError("fail should have aborted already.");
+		}
+	}
+	
 	private static void compareContent(String name, String expectedFile, String actualFile) {
 		String[] expectedLines = expectedFile.split("(\\r?\\n)");
 		String[] actualLines = actualFile.split("(\\r?\\n)");
diff --git a/test/core/src/lombok/CompilerMessage.java b/test/core/src/lombok/CompilerMessage.java
new file mode 100644
index 00000000..d04d17f1
--- /dev/null
+++ b/test/core/src/lombok/CompilerMessage.java
@@ -0,0 +1,50 @@
+package lombok;
+
+public final class CompilerMessage {
+	/** Line Number (starting at 1) */
+	final long line;
+	
+	/** Position is either column number, OR position in file starting from the first byte. */
+	final long position;
+	final boolean isError;
+	final String message;
+	
+	public CompilerMessage(long line, long position, boolean isError, String message) {
+		this.line = line;
+		this.position = position;
+		this.isError = isError;
+		this.message = message;
+	}
+	
+	@Override public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + (isError ? 1231 : 1237);
+		result = prime * result + (int) (line ^ (line >>> 32));
+		result = prime * result + ((message == null) ? 0 : message.hashCode());
+		result = prime * result + (int) (position ^ (position >>> 32));
+		return result;
+	}
+	
+	@Override public boolean equals(Object obj) {
+		if (this == obj) return true;
+		if (obj == null) return false;
+		if (getClass() != obj.getClass()) return false;
+		CompilerMessage other = (CompilerMessage) obj;
+		if (isError != other.isError) return false;
+		if (line != other.line) return false;
+		if (message == null) {
+			if (other.message != null) return false;
+		} else if (!message.equals(other.message)) return false;
+		if (position != other.position) return false;
+		return true;
+	}
+	
+	public CompilerMessageMatcher asCompilerMessageMatcher() {
+		return new CompilerMessageMatcher(line, position, message);
+	}
+	
+	@Override public String toString() {
+		return String.format("%d:%d %s %s", line, position, isError ? "ERROR" : "WARNING", message);
+	}
+}
diff --git a/test/core/src/lombok/CompilerMessageMatcher.java b/test/core/src/lombok/CompilerMessageMatcher.java
new file mode 100644
index 00000000..b7902395
--- /dev/null
+++ b/test/core/src/lombok/CompilerMessageMatcher.java
@@ -0,0 +1,62 @@
+package lombok;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CompilerMessageMatcher {
+	/** Line Number (starting at 1) */
+	private final long line;
+	
+	/** Position is either column number, OR position in file starting from the first byte. */
+	private final long position;
+	private final Collection<String> messageParts;
+	
+	public CompilerMessageMatcher(long line, long position, String message) {
+		this.line = line;
+		this.position = position;
+		this.messageParts = Arrays.asList(message.split("\\s+"));
+	}
+	
+	@Override public String toString() {
+		StringBuilder parts = new StringBuilder();
+		for (String part : messageParts) parts.append(part).append(" ");
+		if (parts.length() > 0) parts.setLength(parts.length() - 1);
+		return String.format("%d:%d %s", line, position, parts);
+	}
+	
+	public boolean matches(CompilerMessage message) {
+		if (message.line != this.line) return false;
+		if (message.position != this.position) return false;
+		for (String token : messageParts) {
+			if (!message.message.contains(token)) return false;
+		}
+		return true;
+	}
+	
+	public static List<CompilerMessageMatcher> readAll(InputStream rawIn) throws IOException {
+		BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, "UTF-8"));
+		List<CompilerMessageMatcher> out = new ArrayList<CompilerMessageMatcher>();
+		for (String line = in.readLine(); line != null; line = in.readLine()) {
+			CompilerMessageMatcher cmm = read(line);
+			if (cmm != null) out.add(cmm);
+		}
+		return out;
+	}
+	
+	private static final Pattern PATTERN = Pattern.compile("^(\\d+):(\\d+) (.*)$");
+	private static CompilerMessageMatcher read(String line) {
+		line = line.trim();
+		if (line.isEmpty()) return null;
+		Matcher m = PATTERN.matcher(line);
+		if (!m.matches()) throw new IllegalArgumentException("Typo in test file: " + line);
+		return new CompilerMessageMatcher(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)), m.group(3));
+	}
+}
diff --git a/test/core/src/lombok/RunTestsViaDelombok.java b/test/core/src/lombok/RunTestsViaDelombok.java
index e58cffc0..16eae8d3 100644
--- a/test/core/src/lombok/RunTestsViaDelombok.java
+++ b/test/core/src/lombok/RunTestsViaDelombok.java
@@ -23,11 +23,13 @@ package lombok;
 
 import java.io.File;
 import java.io.StringWriter;
+import java.util.Collection;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.tools.Diagnostic;
+import javax.tools.Diagnostic.Kind;
 import javax.tools.DiagnosticListener;
 import javax.tools.JavaFileObject;
 
@@ -37,7 +39,7 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 	private Delombok delombok = new Delombok();
 	
 	@Override
-	public void transformCode(final StringBuilder messages, StringWriter result, final File file) throws Throwable {
+	public void transformCode(final Collection<CompilerMessage> messages, StringWriter result, final File file) throws Throwable {
 		delombok.setVerbose(false);
 		delombok.setForceProcess(true);
 		delombok.setCharset("UTF-8");
@@ -49,7 +51,7 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 						"^" + Pattern.quote(file.getAbsolutePath()) +
 						"\\s*:\\s*\\d+\\s*:\\s*(?:warning:\\s*)?(.*)$", Pattern.DOTALL).matcher(msg);
 				if (m.matches()) msg = m.group(1);
-				messages.append(String.format("%d:%d %s %s\n", d.getLineNumber(), d.getColumnNumber(), d.getKind(), msg));
+				messages.add(new CompilerMessage(d.getLineNumber(), d.getColumnNumber(), d.getKind() == Kind.ERROR, msg));
 			}
 		});
 		
diff --git a/test/core/src/lombok/RunTestsViaEcj.java b/test/core/src/lombok/RunTestsViaEcj.java
index 5c573efd..112f07f9 100644
--- a/test/core/src/lombok/RunTestsViaEcj.java
+++ b/test/core/src/lombok/RunTestsViaEcj.java
@@ -25,6 +25,7 @@ import java.io.File;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -84,7 +85,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 	}
 	
 	@Override
-	public void transformCode(final StringBuilder messages, StringWriter result, File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable {
 		final AtomicReference<CompilationResult> compilationResult_ = new AtomicReference<CompilationResult>();
 		final AtomicReference<CompilationUnitDeclaration> compilationUnit_ = new AtomicReference<CompilationUnitDeclaration>();
 		ICompilerRequestor bitbucketRequestor = new ICompilerRequestor() {
@@ -109,7 +110,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 		CategorizedProblem[] problems = compilationResult.getAllProblems();
 		
 		if (problems != null) for (CategorizedProblem p : problems) {
-			messages.append(String.format("%d %s %s\n", p.getSourceLineNumber(), p.isError() ? "error" : p.isWarning() ? "warning" : "unknown", p.getMessage()));
+			messages.add(new CompilerMessage(p.getSourceLineNumber(), p.getSourceStart(), p.isError(), p.getMessage()));
 		}
 		
 		CompilationUnitDeclaration cud = compilationUnit_.get();