5 files changed, 177 insertions, 10 deletions
diff --git a/test/core/src/lombok/AbstractRunTests.java b/test/core/src/lombok/AbstractRunTests.java
index 02c4803a..676426f4 100644
--- a/test/core/src/lombok/AbstractRunTests.java
+++ b/test/core/src/lombok/AbstractRunTests.java
@@ -25,13 +25,19 @@ import static org.junit.Assert.*;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 
 public abstract class AbstractRunTests {
@@ -43,11 +49,23 @@ public abstract class AbstractRunTests {
 	}
 	
 	public boolean compareFile(DirectoryRunner.TestParams params, File file) throws Throwable {
-		StringBuilder messages = new StringBuilder();
+		LinkedHashSet<CompilerMessage> messages = new LinkedHashSet<CompilerMessage>();
 		StringWriter writer = new StringWriter();
 		transformCode(messages, writer, file);
 		String expectedFile = readFile(params.getAfterDirectory(), file, false);
-		String expectedMessages = readFile(params.getMessagesDirectory(), file, true);
+		List<CompilerMessageMatcher> expectedMessages = Collections.emptyList();
+		if (params.getMessagesDirectory() != null) {
+			try {
+				InputStream in = new FileInputStream(new File(params.getMessagesDirectory(), file.getName() + ".messages"));
+				try {
+					expectedMessages = CompilerMessageMatcher.readAll(in);
+				} finally {
+					in.close();
+				}
+			} catch (FileNotFoundException ex) {
+				// That's okay - then we expect no messages, and expectedMessages already gets initialized to the empty list.
+			}
+		}
 		
 		StringReader r = new StringReader(expectedFile);
 		BufferedReader br = new BufferedReader(r);
@@ -58,13 +76,13 @@ public abstract class AbstractRunTests {
 				expectedFile,
 				writer.toString(),
 				expectedMessages,
-				messages.toString(),
+				messages,
 				params.printErrors());
 		
 		return true;
 	}
 	
-	protected abstract void transformCode(StringBuilder message, StringWriter result, File file) throws Throwable;
+	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable;
 	
 	protected String readFile(File file) throws IOException {
 		BufferedReader reader;
@@ -107,7 +125,19 @@ public abstract class AbstractRunTests {
 		}
 	}
 	
-	private void compare(String name, String expectedFile, String actualFile, String expectedMessages, String actualMessages, boolean printErrors) throws Throwable {
+	private static void dumpToFile(File file, Collection<CompilerMessage> content) throws IOException {
+		FileOutputStream fos = new FileOutputStream(file);
+		try {
+			for (CompilerMessage message : content) {
+				fos.write(message.asCompilerMessageMatcher().toString().getBytes("UTF-8"));
+				fos.write('\n');
+			}
+		} finally {
+			fos.close();
+		}
+	}
+	
+	private void compare(String name, String expectedFile, String actualFile, List<CompilerMessageMatcher> expectedMessages, LinkedHashSet<CompilerMessage> actualMessages, boolean printErrors) throws Throwable {
 		try {
 			compareContent(name, expectedFile, actualFile);
 		} catch (Throwable e) {
@@ -129,8 +159,9 @@ public abstract class AbstractRunTests {
 			}
 			throw e;
 		}
+		
 		try {
-			compareContent(name, expectedMessages, actualMessages);
+			compareMessages(name, expectedMessages, actualMessages);
 		} catch (Throwable e) {
 			if (printErrors) {
 				System.out.println("***** " + name + " *****");
@@ -148,6 +179,27 @@ public abstract class AbstractRunTests {
 		}
 	}
 	
+	private static void compareMessages(String name, List<CompilerMessageMatcher> expected, LinkedHashSet<CompilerMessage> actual) {
+		Iterator<CompilerMessageMatcher> expectedIterator = expected.iterator();
+		Iterator<CompilerMessage> actualIterator = actual.iterator();
+		
+		while (true) {
+			boolean exHasNext = expectedIterator.hasNext();
+			boolean acHasNext = actualIterator.hasNext();
+			if (!exHasNext && !acHasNext) break;
+			if (exHasNext && acHasNext) {
+				CompilerMessageMatcher cmm = expectedIterator.next();
+				CompilerMessage cm = actualIterator.next();
+				if (cmm.matches(cm)) continue;
+				fail(String.format("[%s] Expected message '%s' but got message '%s'", name, cmm, cm));
+				throw new AssertionError("fail should have aborted already.");
+			}
+			if (exHasNext) fail(String.format("[%s] Expected message '%s' but ran out of actual messages", name, expectedIterator.next()));
+			if (acHasNext) fail(String.format("[%s] Unexpected message: %s", name, actualIterator.next()));
+			throw new AssertionError("fail should have aborted already.");
+		}
+	}
+	
 	private static void compareContent(String name, String expectedFile, String actualFile) {
 		String[] expectedLines = expectedFile.split("(\\r?\\n)");
 		String[] actualLines = actualFile.split("(\\r?\\n)");
diff --git a/test/core/src/lombok/CompilerMessage.java b/test/core/src/lombok/CompilerMessage.java
new file mode 100644
index 00000000..d04d17f1
--- /dev/null
+++ b/test/core/src/lombok/CompilerMessage.java
@@ -0,0 +1,50 @@
+package lombok;
+
+public final class CompilerMessage {
+	/** Line Number (starting at 1) */
+	final long line;
+	
+	/** Position is either column number, OR position in file starting from the first byte. */
+	final long position;
+	final boolean isError;
+	final String message;
+	
+	public CompilerMessage(long line, long position, boolean isError, String message) {
+		this.line = line;
+		this.position = position;
+		this.isError = isError;
+		this.message = message;
+	}
+	
+	@Override public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + (isError ? 1231 : 1237);
+		result = prime * result + (int) (line ^ (line >>> 32));
+		result = prime * result + ((message == null) ? 0 : message.hashCode());
+		result = prime * result + (int) (position ^ (position >>> 32));
+		return result;
+	}
+	
+	@Override public boolean equals(Object obj) {
+		if (this == obj) return true;
+		if (obj == null) return false;
+		if (getClass() != obj.getClass()) return false;
+		CompilerMessage other = (CompilerMessage) obj;
+		if (isError != other.isError) return false;
+		if (line != other.line) return false;
+		if (message == null) {
+			if (other.message != null) return false;
+		} else if (!message.equals(other.message)) return false;
+		if (position != other.position) return false;
+		return true;
+	}
+	
+	public CompilerMessageMatcher asCompilerMessageMatcher() {
+		return new CompilerMessageMatcher(line, position, message);
+	}
+	
+	@Override public String toString() {
+		return String.format("%d:%d %s %s", line, position, isError ? "ERROR" : "WARNING", message);
+	}
+}
diff --git a/test/core/src/lombok/CompilerMessageMatcher.java b/test/core/src/lombok/CompilerMessageMatcher.java
new file mode 100644
index 00000000..b7902395
--- /dev/null
+++ b/test/core/src/lombok/CompilerMessageMatcher.java
@@ -0,0 +1,62 @@
+package lombok;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CompilerMessageMatcher {
+	/** Line Number (starting at 1) */
+	private final long line;
+	
+	/** Position is either column number, OR position in file starting from the first byte. */
+	private final long position;
+	private final Collection<String> messageParts;
+	
+	public CompilerMessageMatcher(long line, long position, String message) {
+		this.line = line;
+		this.position = position;
+		this.messageParts = Arrays.asList(message.split("\\s+"));
+	}
+	
+	@Override public String toString() {
+		StringBuilder parts = new StringBuilder();
+		for (String part : messageParts) parts.append(part).append(" ");
+		if (parts.length() > 0) parts.setLength(parts.length() - 1);
+		return String.format("%d:%d %s", line, position, parts);
+	}
+	
+	public boolean matches(CompilerMessage message) {
+		if (message.line != this.line) return false;
+		if (message.position != this.position) return false;
+		for (String token : messageParts) {
+			if (!message.message.contains(token)) return false;
+		}
+		return true;
+	}
+	
+	public static List<CompilerMessageMatcher> readAll(InputStream rawIn) throws IOException {
+		BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, "UTF-8"));
+		List<CompilerMessageMatcher> out = new ArrayList<CompilerMessageMatcher>();
+		for (String line = in.readLine(); line != null; line = in.readLine()) {
+			CompilerMessageMatcher cmm = read(line);
+			if (cmm != null) out.add(cmm);
+		}
+		return out;
+	}
+	
+	private static final Pattern PATTERN = Pattern.compile("^(\\d+):(\\d+) (.*)$");
+	private static CompilerMessageMatcher read(String line) {
+		line = line.trim();
+		if (line.isEmpty()) return null;
+		Matcher m = PATTERN.matcher(line);
+		if (!m.matches()) throw new IllegalArgumentException("Typo in test file: " + line);
+		return new CompilerMessageMatcher(Integer.parseInt(m.group(1)), Integer.parseInt(m.group(2)), m.group(3));
+	}
+}
diff --git a/test/core/src/lombok/RunTestsViaDelombok.java b/test/core/src/lombok/RunTestsViaDelombok.java
index e58cffc0..16eae8d3 100644
--- a/test/core/src/lombok/RunTestsViaDelombok.java
+++ b/test/core/src/lombok/RunTestsViaDelombok.java
@@ -23,11 +23,13 @@ package lombok;
 
 import java.io.File;
 import java.io.StringWriter;
+import java.util.Collection;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.tools.Diagnostic;
+import javax.tools.Diagnostic.Kind;
 import javax.tools.DiagnosticListener;
 import javax.tools.JavaFileObject;
 
@@ -37,7 +39,7 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 	private Delombok delombok = new Delombok();
 	
 	@Override
-	public void transformCode(final StringBuilder messages, StringWriter result, final File file) throws Throwable {
+	public void transformCode(final Collection<CompilerMessage> messages, StringWriter result, final File file) throws Throwable {
 		delombok.setVerbose(false);
 		delombok.setForceProcess(true);
 		delombok.setCharset("UTF-8");
@@ -49,7 +51,7 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 						"^" + Pattern.quote(file.getAbsolutePath()) +
 						"\\s*:\\s*\\d+\\s*:\\s*(?:warning:\\s*)?(.*)$", Pattern.DOTALL).matcher(msg);
 				if (m.matches()) msg = m.group(1);
-				messages.append(String.format("%d:%d %s %s\n", d.getLineNumber(), d.getColumnNumber(), d.getKind(), msg));
+				messages.add(new CompilerMessage(d.getLineNumber(), d.getColumnNumber(), d.getKind() == Kind.ERROR, msg));
 			}
 		});
 		
diff --git a/test/core/src/lombok/RunTestsViaEcj.java b/test/core/src/lombok/RunTestsViaEcj.java
index 5c573efd..112f07f9 100644
--- a/test/core/src/lombok/RunTestsViaEcj.java
+++ b/test/core/src/lombok/RunTestsViaEcj.java
@@ -25,6 +25,7 @@ import java.io.File;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -84,7 +85,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 	}
 	
 	@Override
-	public void transformCode(final StringBuilder messages, StringWriter result, File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable {
 		final AtomicReference<CompilationResult> compilationResult_ = new AtomicReference<CompilationResult>();
 		final AtomicReference<CompilationUnitDeclaration> compilationUnit_ = new AtomicReference<CompilationUnitDeclaration>();
 		ICompilerRequestor bitbucketRequestor = new ICompilerRequestor() {
@@ -109,7 +110,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 		CategorizedProblem[] problems = compilationResult.getAllProblems();
 		
 		if (problems != null) for (CategorizedProblem p : problems) {
-			messages.append(String.format("%d %s %s\n", p.getSourceLineNumber(), p.isError() ? "error" : p.isWarning() ? "warning" : "unknown", p.getMessage()));
+			messages.add(new CompilerMessage(p.getSourceLineNumber(), p.getSourceStart(), p.isError(), p.getMessage()));
 		}
 		
 		CompilationUnitDeclaration cud = compilationUnit_.get();