[issue #759] delombok now plays nicely with exotic characters, encodings, and backslash U escapes.

14 files changed, 116 insertions, 17 deletions

diff --git a/doc/changelog.markdown b/doc/changelog.markdown
index e4a004af..5974cf07 100644
--- a/doc/changelog.markdown
+++ b/doc/changelog.markdown
@@ -3,6 +3,7 @@ Lombok Changelog
 
 ### v1.14.9.shadow "<del>Edgy</del> Shadowy Guinea Pig"
 * Added a launcher to the lombok boot process which removes the need for `-Xbootclasspath` to be in your `eclipse.ini` file, and removes all non-public API and third party dependencies (such as ASM) from the lombok jar, thus removing them from your IDE's auto complete offerings in any project that uses lombok. For those debugging lombok, the launcher enables hot code replace which makes debugging a lot easier, as previously one was required to shut down the IDE, rebuild the jar, and relaunch. Add `-Dshadow.override.lombok=/path/to/lombok/bin` to the launch target for hot code replace.
+* BUGFIX: delombok: Using exotic characters in your source files would overzealously backslash-u escape them. Now, all characters are printed unescaped, assuming your chosen encoding can support them. Otherwise, they are escaped. [Issue #759](https://code.google.com/p/projectlombok/issues/detail?id=759)
 
 ### v1.14.8 (September 15th, 2014)
 * PERFORMANCE: The configuration system typically hit the filesystem twice per read configuration key instead of hardly ever. This is a continuation of [Issue #682](https://code.google.com/p/projectlombok/issues/detail?id=682).
diff --git a/src/delombok/lombok/delombok/Delombok.java b/src/delombok/lombok/delombok/Delombok.java
index f64e36a1..059bb004 100644
--- a/src/delombok/lombok/delombok/Delombok.java
+++ b/src/delombok/lombok/delombok/Delombok.java
@@ -509,7 +509,7 @@ public class Delombok {
 			DelombokResult result = new DelombokResult(catcher.getComments(unit), unit, force || options.isChanged(unit), fps);
 			if (verbose) feedback.printf("File: %s [%s]\n", unit.sourcefile.getName(), result.isChanged() ? "delomboked" : "unchanged");
 			Writer rawWriter;
-			if (presetWriter != null) rawWriter = presetWriter;
+			if (presetWriter != null) rawWriter = createUnicodeEscapeWriter(presetWriter);
 			else if (output == null) rawWriter = createStandardOutWriter();
 			else rawWriter = createFileWriter(output, baseMap.get(unit), unit.sourcefile.toUri());
 			BufferedWriter writer = new BufferedWriter(rawWriter);
@@ -605,6 +605,10 @@ public class Delombok {
 		return createUnicodeEscapeWriter(System.out);
 	}
 	
+	private Writer createUnicodeEscapeWriter(Writer writer) {
+		return new UnicodeEscapeWriter(writer, charset);
+	}
+	
 	private Writer createUnicodeEscapeWriter(OutputStream out) {
 		return new UnicodeEscapeWriter(new OutputStreamWriter(out, charset), charset);
 	}
diff --git a/src/delombok/lombok/delombok/PrettyCommentsPrinter.java b/src/delombok/lombok/delombok/PrettyCommentsPrinter.java
index 9dd7e40e..72a8ff59 100644
--- a/src/delombok/lombok/delombok/PrettyCommentsPrinter.java
+++ b/src/delombok/lombok/delombok/PrettyCommentsPrinter.java
@@ -104,7 +104,6 @@ import com.sun.tools.javac.tree.JCTree.LetExpr;
 import com.sun.tools.javac.tree.JCTree.TypeBoundKind;
 import com.sun.tools.javac.tree.TreeInfo;
 import com.sun.tools.javac.tree.TreeScanner;
-import com.sun.tools.javac.util.Convert;
 import com.sun.tools.javac.util.List;
 import com.sun.tools.javac.util.Name;
 import com.sun.tools.javac.util.Position;
@@ -209,7 +208,6 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 	
 	private void consumeComments(int until, JCTree tree) throws IOException {
 		boolean prevNewLine = onNewLine;
-		boolean found = false;
 		CommentInfo head = comments.head;
 		while (comments.nonEmpty() && head.pos < until) {
 			printComment(head);
@@ -355,7 +353,7 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 		}
 		needsSpace = false;
 		
-		out.write(Convert.escapeUnicode(s.toString()));
+		out.write(s.toString());
 		
 		onNewLine = false;
 		aligned = false;
@@ -1413,16 +1411,51 @@ public class PrettyCommentsPrinter extends JCTree.Visitor {
 			else if (CTC_FLOAT.equals(typeTag)) print(tree.value + "F");
 			else if (CTC_DOUBLE.equals(typeTag)) print(tree.value.toString());
 			else if (CTC_CHAR.equals(typeTag)) {
-				print("\'" + Convert.quote(String.valueOf((char)((Number)tree.value).intValue())) + "\'");
+				print("\'" + quoteChar((char)((Number)tree.value).intValue()) + "\'");
 			}
 			else if (CTC_BOOLEAN.equals(typeTag)) print(((Number)tree.value).intValue() == 1 ? "true" : "false");
 			else if (CTC_BOT.equals(typeTag)) print("null");
-			else print("\"" + Convert.quote(tree.value.toString()) + "\"");
+			else print("\"" + quoteChars(tree.value.toString()) + "\"");
 		} catch (IOException e) {
 			throw new UncheckedIOException(e);
 		}
 	}
 	
+	public static String quoteChars(String s) {
+		StringBuilder buf = new StringBuilder();
+		for (int i = 0; i < s.length(); i++) {
+			buf.append(quoteChar(s.charAt(i)));
+		}
+		return buf.toString();
+	}
+	
+	/**
+	 * Escapes a character if it has an escape sequence or is non-printable
+	 * ASCII. Leaves non-ASCII characters alone.
+	 */
+	public static String quoteChar(char ch) {
+		switch (ch) {
+		case '\b':
+			return "\\b";
+		case '\f':
+			return "\\f";
+		case '\n':
+			return "\\n";
+		case '\r':
+			return "\\r";
+		case '\t':
+			return "\\t";
+		case '\'':
+			return "\\'";
+		case '\"':
+			return "\\\"";
+		case '\\':
+			return "\\\\";
+		default:
+			return ch < 32 ? String.format("\\%03o", (int) ch) : String.valueOf(ch);
+		}
+	}
+	
 	public void visitTypeIdent(JCPrimitiveTypeTree tree) {
 		TypeTag typetag = typeTag(tree);
 		try {
diff --git a/src/delombok/lombok/delombok/UnicodeEscapeWriter.java b/src/delombok/lombok/delombok/UnicodeEscapeWriter.java
index 95b55078..9c2ef190 100644
--- a/src/delombok/lombok/delombok/UnicodeEscapeWriter.java
+++ b/src/delombok/lombok/delombok/UnicodeEscapeWriter.java
@@ -63,6 +63,6 @@ public class UnicodeEscapeWriter extends Writer {
 	}
 	
 	protected void writeUnicodeEscape(char c) throws IOException {
-		writer.write("\\u" + Integer.toHexString(c));
+		writer.write(String.format("\\u%04x", (int) c));
 	}
 }
\ No newline at end of file
diff --git a/test/core/src/lombok/AbstractRunTests.java b/test/core/src/lombok/AbstractRunTests.java
index a1f535b4..85d4d4f3 100644
--- a/test/core/src/lombok/AbstractRunTests.java
+++ b/test/core/src/lombok/AbstractRunTests.java
@@ -75,7 +75,7 @@ public abstract class AbstractRunTests {
 			}
 		});
 		
-		transformCode(messages, writer, file);
+		transformCode(messages, writer, file, sourceDirectives.getSpecifiedEncoding());
 		
 		compare(file.getName(), expected, writer.toString(), messages, params.printErrors(), sourceDirectives.isSkipCompareContent() || expected.isSkipCompareContent());
 		return true;
@@ -97,7 +97,7 @@ public abstract class AbstractRunTests {
 		return 8;
 	}
 	
-	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable;
+	protected abstract void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file, String encoding) throws Throwable;
 	
 	protected String readFile(File file) throws IOException {
 		BufferedReader reader;
diff --git a/test/core/src/lombok/LombokTestSource.java b/test/core/src/lombok/LombokTestSource.java
index 402001e9..f31d7be7 100644
--- a/test/core/src/lombok/LombokTestSource.java
+++ b/test/core/src/lombok/LombokTestSource.java
@@ -49,7 +49,8 @@ public class LombokTestSource {
 	private final boolean skipCompareContent;
 	private final int versionLowerLimit, versionUpperLimit;
 	private final ConfigurationResolver configuration;
-	
+	private final String specifiedEncoding;
+
 	public boolean versionWithinLimit(int version) {
 		return version >= versionLowerLimit && version <= versionUpperLimit;
 	}
@@ -74,6 +75,10 @@ public class LombokTestSource {
 		return skipCompareContent;
 	}
 	
+	public String getSpecifiedEncoding() {
+		return specifiedEncoding;
+	}
+	
 	public ConfigurationResolver getConfiguration() {
 		return configuration;
 	}
@@ -123,6 +128,7 @@ public class LombokTestSource {
 		int versionUpper = Integer.MAX_VALUE;
 		boolean ignore = false;
 		boolean skipCompareContent = false;
+		String encoding = null;
 		
 		for (String directive : directives) {
 			directive = directive.trim();
@@ -154,10 +160,15 @@ public class LombokTestSource {
 				continue;
 			}
 			
+			if (lc.startsWith("encoding:")) {
+				encoding = directive.substring(9).trim();
+				continue;
+			}
+			
 			Assert.fail("Directive line \"" + directive + "\" in '" + file.getAbsolutePath() + "' invalid: unrecognized directive.");
 			throw new RuntimeException();
 		}
-		
+		this.specifiedEncoding = encoding;
 		this.versionLowerLimit = versionLower;
 		this.versionUpperLimit = versionUpper;
 		this.ignore = ignore;
@@ -194,13 +205,17 @@ public class LombokTestSource {
 	}
 	
 	public static LombokTestSource read(File sourceFolder, File messagesFolder, String fileName) throws IOException {
+		return read0(sourceFolder, messagesFolder, fileName, "UTF-8");
+	}
+	
+	private static LombokTestSource read0(File sourceFolder, File messagesFolder, String fileName, String encoding) throws IOException {
 		StringBuilder content = null;
 		List<String> directives = new ArrayList<String>();
 		
 		File sourceFile = new File(sourceFolder, fileName);
 		if (sourceFile.exists()) {
 			@Cleanup val rawIn = new FileInputStream(sourceFile);
-			BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, "UTF-8"));
+			BufferedReader in = new BufferedReader(new InputStreamReader(rawIn, encoding));
 			for (String i = in.readLine(); i != null; i = in.readLine()) {
 				if (content != null) {
 					content.append(i).append("\n");
@@ -234,6 +249,15 @@ public class LombokTestSource {
 			}
 		}
 		
-		return new LombokTestSource(sourceFile, content.toString(), messages, directives);
+		LombokTestSource source = new LombokTestSource(sourceFile, content.toString(), messages, directives);
+		String specifiedEncoding = source.getSpecifiedEncoding();
+		
+		// The source file has an 'encoding' header to test encoding issues. Of course, reading the encoding header
+		// requires knowing the encoding of the file first. In practice we get away with it, because UTF-8 and US-ASCII are compatible enough.
+		// The fix is therefore to read in as UTF-8 initially, and if the file requests that it should be read as another encoding, toss it all
+		// and reread that way.
+		
+		if (specifiedEncoding == null || specifiedEncoding.equalsIgnoreCase(encoding)) return source;
+		return read0(sourceFolder, messagesFolder, fileName, specifiedEncoding);
 	}
 }
diff --git a/test/core/src/lombok/RunTestsViaDelombok.java b/test/core/src/lombok/RunTestsViaDelombok.java
index 6a08642b..1482c865 100644
--- a/test/core/src/lombok/RunTestsViaDelombok.java
+++ b/test/core/src/lombok/RunTestsViaDelombok.java
@@ -34,10 +34,10 @@ public class RunTestsViaDelombok extends AbstractRunTests {
 	private Delombok delombok = new Delombok();
 	
 	@Override
-	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, final File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, final File file, String encoding) throws Throwable {
 		delombok.setVerbose(false);
 		delombok.setForceProcess(true);
-		delombok.setCharset("UTF-8");
+		delombok.setCharset(encoding == null ? "UTF-8" : encoding);
 		
 		delombok.setDiagnosticsListener(new CapturingDiagnosticListener(file, messages));
 		
diff --git a/test/core/src/lombok/RunTestsViaEcj.java b/test/core/src/lombok/RunTestsViaEcj.java
index f4584493..1a94baf1 100644
--- a/test/core/src/lombok/RunTestsViaEcj.java
+++ b/test/core/src/lombok/RunTestsViaEcj.java
@@ -93,7 +93,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 	}
 	
 	@Override
-	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file) throws Throwable {
+	public void transformCode(Collection<CompilerMessage> messages, StringWriter result, File file, String encoding) throws Throwable {
 		final AtomicReference<CompilationResult> compilationResult_ = new AtomicReference<CompilationResult>();
 		final AtomicReference<CompilationUnitDeclaration> compilationUnit_ = new AtomicReference<CompilationUnitDeclaration>();
 		ICompilerRequestor bitbucketRequestor = new ICompilerRequestor() {
@@ -103,7 +103,7 @@ public class RunTestsViaEcj extends AbstractRunTests {
 		};
 		
 		String source = readFile(file);
-		final CompilationUnit sourceUnit = new CompilationUnit(source.toCharArray(), file.getName(), "UTF-8");
+		final CompilationUnit sourceUnit = new CompilationUnit(source.toCharArray(), file.getName(), encoding == null ? "UTF-8" : encoding);
 		
 		Compiler ecjCompiler = new Compiler(createFileSystem(file), ecjErrorHandlingPolicy(), ecjCompilerOptions(), bitbucketRequestor, new DefaultProblemFactory(Locale.ENGLISH)) {
 			@Override protected synchronized void addCompilationUnit(ICompilationUnit inUnit, CompilationUnitDeclaration parsedUnit) {
diff --git a/test/transform/resource/after-delombok/EncodingUsAscii.java b/test/transform/resource/after-delombok/EncodingUsAscii.java
new file mode 100644
index 00000000..a9e6b1a5
--- /dev/null
+++ b/test/transform/resource/after-delombok/EncodingUsAscii.java
@@ -0,0 +1,9 @@
+//ENCODING: US-ASCII
+class EncodingUsAscii {
+	String foo\u0e51\u0e51 = "\016\t\b ";
+	@java.lang.Override
+	@java.lang.SuppressWarnings("all")
+	public java.lang.String toString() {
+		return "EncodingUsAscii(foo\u0e51\u0e51=" + this.foo\u0e51\u0e51 + ")";
+	}
+}
\ No newline at end of file
diff --git a/test/transform/resource/after-delombok/EncodingUtf8.java b/test/transform/resource/after-delombok/EncodingUtf8.java
new file mode 100644
index 00000000..9ae3e30a
--- /dev/null
+++ b/test/transform/resource/after-delombok/EncodingUtf8.java
@@ -0,0 +1,8 @@
+class EncodingUtf8 {
+	String foo๑๑ = "\016\t\b ";
+	@java.lang.Override
+	@java.lang.SuppressWarnings("all")
+	public java.lang.String toString() {
+		return "EncodingUtf8(foo๑๑=" + this.foo๑๑ + ")";
+	}
+}
\ No newline at end of file
diff --git a/test/transform/resource/after-ecj/EncodingUsAscii.java b/test/transform/resource/after-ecj/EncodingUsAscii.java
new file mode 100644
index 00000000..b66feb0c
--- /dev/null
+++ b/test/transform/resource/after-ecj/EncodingUsAscii.java
@@ -0,0 +1 @@
+//ignore: This test serves to check what happens with 'weird' characters when you use delombok. It's just not relevant for ecj.
diff --git a/test/transform/resource/after-ecj/EncodingUtf8.java b/test/transform/resource/after-ecj/EncodingUtf8.java
new file mode 100644
index 00000000..5a7a1644
--- /dev/null
+++ b/test/transform/resource/after-ecj/EncodingUtf8.java
@@ -0,0 +1,9 @@
+@lombok.ToString class EncodingUtf8 {
+  String foo๑๑ = "\t\b ";
+  EncodingUtf8() {
+    super();
+  }
+  public @java.lang.Override @java.lang.SuppressWarnings("all") java.lang.String toString() {
+    return (("EncodingUtf8(foo๑๑=" + this.foo๑๑) + ")");
+  }
+}
\ No newline at end of file
diff --git a/test/transform/resource/before/EncodingUsAscii.java b/test/transform/resource/before/EncodingUsAscii.java
new file mode 100644
index 00000000..dbcd150b
--- /dev/null
+++ b/test/transform/resource/before/EncodingUsAscii.java
@@ -0,0 +1,5 @@
+//ENCODING: US-ASCII
+@lombok.ToString
+class EncodingUsAscii {
+	String foo\u0e51\u0e51 = "\u000e	\10 ";
+}
\ No newline at end of file
diff --git a/test/transform/resource/before/EncodingUtf8.java b/test/transform/resource/before/EncodingUtf8.java
new file mode 100644
index 00000000..75b0ee00
--- /dev/null
+++ b/test/transform/resource/before/EncodingUtf8.java
@@ -0,0 +1,5 @@
+//ENCODING: UTF-8
+@lombok.ToString
+class EncodingUtf8 {
+	String foo\u0e51๑ = "\u000e	\10 ";
+}
\ No newline at end of file