2 files changed, 56 insertions, 21 deletions

diff --git a/src/utils/lombok/javac/java8/CommentCollectingScannerFactory.java b/src/utils/lombok/javac/java8/CommentCollectingScannerFactory.java
index cb0d2e12..f29f501b 100644
--- a/src/utils/lombok/javac/java8/CommentCollectingScannerFactory.java
+++ b/src/utils/lombok/javac/java8/CommentCollectingScannerFactory.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2011-2019 The Project Lombok Authors.
+ * Copyright (C) 2011-2021 The Project Lombok Authors.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -72,18 +72,31 @@ public class CommentCollectingScannerFactory extends ScannerFactory {
 		super(context);
 	}
 	
+	@SuppressWarnings("all")
 	@Override
 	public Scanner newScanner(CharSequence input, boolean keepDocComments) {
-		if (input instanceof CharBuffer) {
-			CharBuffer buf = (CharBuffer) input;
-			return new CommentCollectingScanner(this, new CommentCollectingTokenizer(this, buf, findTextBlocks));
+		char[] array;
+		int limit;
+		if (input instanceof CharBuffer && ((CharBuffer) input).hasArray()) {
+			CharBuffer cb = (CharBuffer) input;
+			cb.compact().flip();
+			array = cb.array();
+			limit = cb.limit();
+		} else {
+			array = input.toString().toCharArray();
+			limit = array.length;
+		}
+		if (array.length == limit) {
+			// work around a bug where the last comment in a file falls away in this case.
+			char[] d = new char[limit + 1];
+			System.arraycopy(array, 0, d, 0, limit);
+			array = d;
 		}
-		char[] array = input.toString().toCharArray();
-		return newScanner(array, array.length, keepDocComments);
+		return newScanner(array, limit, keepDocComments);
 	}
 	
 	@Override
 	public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
-		return new CommentCollectingScanner(this, new CommentCollectingTokenizer(this, input, inputLength, findTextBlocks));
+		return new CommentCollectingScanner(this, CommentCollectingTokenizer.create(this, input, inputLength, findTextBlocks));
 	}
 }
diff --git a/src/utils/lombok/javac/java8/CommentCollectingTokenizer.java b/src/utils/lombok/javac/java8/CommentCollectingTokenizer.java
index d7b1d569..4a31fc81 100644
--- a/src/utils/lombok/javac/java8/CommentCollectingTokenizer.java
+++ b/src/utils/lombok/javac/java8/CommentCollectingTokenizer.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013-2020 The Project Lombok Authors.
+ * Copyright (C) 2013-2021 The Project Lombok Authors.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,36 +23,51 @@ package lombok.javac.java8;
 
 import java.nio.CharBuffer;
 
-import lombok.javac.CommentInfo;
-import lombok.javac.CommentInfo.EndConnection;
-import lombok.javac.CommentInfo.StartConnection;
-
 import com.sun.tools.javac.parser.JavaTokenizer;
 import com.sun.tools.javac.parser.ScannerFactory;
 import com.sun.tools.javac.parser.Tokens.Comment;
-import com.sun.tools.javac.parser.Tokens.Token;
 import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
+import com.sun.tools.javac.parser.Tokens.Token;
 import com.sun.tools.javac.parser.UnicodeReader;
 import com.sun.tools.javac.util.List;
 import com.sun.tools.javac.util.ListBuffer;
 
+import lombok.javac.CommentInfo;
+import lombok.javac.CommentInfo.EndConnection;
+import lombok.javac.CommentInfo.StartConnection;
+
 class CommentCollectingTokenizer extends JavaTokenizer {
+	
+	private static final boolean tokenizerIsUnicodeReader = JavaTokenizer.class.getSuperclass().getSimpleName().equals("UnicodeReader");
+	
 	private int prevEndPosition = 0;
 	private final ListBuffer<CommentInfo> comments = new ListBuffer<CommentInfo>();
 	private final ListBuffer<Integer> textBlockStarts;
 	private int endComment = 0;
 	
-	CommentCollectingTokenizer(ScannerFactory fac, char[] buf, int inputLength, boolean findTextBlocks) {
+	static CommentCollectingTokenizer create(ScannerFactory fac, char[] buf, int inputLength, boolean findTextBlocks) {
+		if (tokenizerIsUnicodeReader) {
+			return new CommentCollectingTokenizer(fac, buf, inputLength, findTextBlocks, true);
+		}
+		return new CommentCollectingTokenizer(fac, buf, inputLength, findTextBlocks);
+	}
+	
+	// pre java 16
+	private CommentCollectingTokenizer(ScannerFactory fac, char[] buf, int inputLength, boolean findTextBlocks) {
 		super(fac, new PositionUnicodeReader(fac, buf, inputLength));
 		textBlockStarts = findTextBlocks ? new ListBuffer<Integer>() : null;
 	}
 
-	CommentCollectingTokenizer(ScannerFactory fac, CharBuffer buf, boolean findTextBlocks) {
-		super(fac, new PositionUnicodeReader(fac, buf));
+	// from java 16
+	private CommentCollectingTokenizer(ScannerFactory fac, char[] buf, int inputLength, boolean findTextBlocks, boolean java16Signature) {
+		super(fac, buf, inputLength);
 		textBlockStarts = findTextBlocks ? new ListBuffer<Integer>() : null;
 	}
 	
 	int pos() {
+		if (tokenizerIsUnicodeReader) {
+			return position();
+		}
 		return ((PositionUnicodeReader) reader).pos();
 	}
 	
@@ -60,8 +75,8 @@ class CommentCollectingTokenizer extends JavaTokenizer {
 		Token token = super.readToken();
 		prevEndPosition = pos();
 		if (textBlockStarts != null && (prevEndPosition - token.pos > 5) && token.getClass().getName().endsWith("$StringToken")) {
-			char[] start = reader.getRawCharacters(token.pos, token.pos + 3);
-			if (start[0] == '"' && start[1] == '"' && start[2] == '"') textBlockStarts.add(token.pos);
+			char[] start = reader().getRawCharacters(token.pos, token.pos + 3);
+			if (start[0] == '"' && start[1] == '"' && start[2] == '"') textBlockStarts.append(token.pos);
 		}
 		return token;
 	}
@@ -70,7 +85,7 @@ class CommentCollectingTokenizer extends JavaTokenizer {
 	protected Comment processComment(int pos, int endPos, CommentStyle style) {
 		int prevEndPos = Math.max(prevEndPosition, endComment);
 		endComment = endPos;
-		String content = new String(reader.getRawCharacters(pos, endPos));
+		String content = new String(reader().getRawCharacters(pos, endPos));
 		StartConnection start = determineStartConnection(prevEndPos, pos);
 		EndConnection end = determineEndConnection(endPos); 
 
@@ -85,7 +100,7 @@ class CommentCollectingTokenizer extends JavaTokenizer {
 		for (int i = pos;; i++) {
 			char c;
 			try {
-				c = reader.getRawCharacters(i, i + 1)[0];
+				c = reader().getRawCharacters(i, i + 1)[0];
 			} catch (IndexOutOfBoundsException e) {
 				c = '\n';
 			}
@@ -104,7 +119,7 @@ class CommentCollectingTokenizer extends JavaTokenizer {
 		if (from == to) {
 			return StartConnection.DIRECT_AFTER_PREVIOUS;
 		}
-		char[] between = reader.getRawCharacters(from, to);
+		char[] between = reader().getRawCharacters(from, to);
 		if (isNewLine(between[between.length - 1])) {
 			return StartConnection.START_OF_LINE;
 		}
@@ -128,6 +143,13 @@ class CommentCollectingTokenizer extends JavaTokenizer {
 		return textBlockStarts == null ? List.<Integer>nil() : textBlockStarts.toList();
 	}
 	
+	private UnicodeReader reader() {
+		if (tokenizerIsUnicodeReader) {
+			return (UnicodeReader) (Object) this;
+		}
+		return reader;
+	}
+	
 	static class PositionUnicodeReader extends UnicodeReader {
 		protected PositionUnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
 			super(sf, input, inputLength);