[build] Uploading edge repo to cloudflare R2 bucket straight from ant now possible

9 files changed, 246 insertions, 87 deletions

diff --git a/.gitignore b/.gitignore
index d4285573..c9dcccb5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,6 @@
 /google.properties
 /debug
 /*.launch
-/ssh.configuration
 /findbugsReport.html
 /lib
 /.settings
@@ -24,4 +23,5 @@
 /website/lombokSupporters
 /pom.xml
 /jvm.locations
-/testenv
\ No newline at end of file
+/testenv
+/gpg.keyring
diff --git a/buildScripts/eclipse-p2.ant.xml b/buildScripts/eclipse-p2.ant.xml
index 75f6c4eb..baf48c7a 100644
--- a/buildScripts/eclipse-p2.ant.xml
+++ b/buildScripts/eclipse-p2.ant.xml
@@ -115,7 +115,7 @@ This buildfile is part of projectlombok.org. It is responsible for building the
 		</tar>
 	</target>
 	
-	<target name="eclipsep2.publish" depends="setup.ssh, eclipsep2.pack">
+	<target name="eclipsep2.publish" depends="eclipsep2.pack">
 		<ivy:scpUpload
 			from="dist/eclipse-p2.tar.bz2"
 			to="/data/lombok/staging"
diff --git a/buildScripts/ivy.xml b/buildScripts/ivy.xml
index adab33b4..f3816999 100644
--- a/buildScripts/ivy.xml
+++ b/buildScripts/ivy.xml
@@ -65,6 +65,7 @@
 		<dependency org="de.java2html" name="java2html" rev="5.0" conf="buildtools->default" />
 		<dependency org="org.freemarker" name="freemarker" rev="2.3.28" conf="buildtools->default" />
 		<dependency org="com.sparkjava" name="spark-core" rev="2.9.2" conf="buildtools->default" />
+		<dependency org="software.amazon.awssdk" name="s3" rev="2.19.29" conf="buildtools->default" />
 		
 		<dependency org="org.eclipse.jgit" name="org.eclipse.jgit.ant" rev="5.2.0.201812061821-r" conf="supporters->default" />
 		<dependency org="org.eclipse.jgit" name="org.eclipse.jgit" rev="5.2.0.201812061821-r" conf="supporters->default" />
diff --git a/buildScripts/mapstructBinding.ant.xml b/buildScripts/mapstructBinding.ant.xml
index d7c52dc3..eb80dc87 100644
--- a/buildScripts/mapstructBinding.ant.xml
+++ b/buildScripts/mapstructBinding.ant.xml
@@ -96,7 +96,7 @@ exists as a separate dependency solely because it is itself dependent on both lo
 	<target name="mapstructBinding.pack" depends="dist,-mapstructBinding.jar,-mapstructBinding.doc,-mapstructBinding.src">
 	</target>
 	
-	<target name="mapstructBinding.publish" depends="mapstructBinding.pack, setup.ssh">
+	<target name="mapstructBinding.publish" depends="mapstructBinding.pack">
 		<tar destfile="dist/mavenPublish-mapstructBinding.tar.bz2" compression="bzip2">
 		<tarfileset dir="dist">
 			<include name="lombok-mapstruct-binding-${mapstruct-binding.version}.jar" />
diff --git a/buildScripts/maven.ant.xml b/buildScripts/maven.ant.xml
index fe64a611..a09e09de 100644
--- a/buildScripts/maven.ant.xml
+++ b/buildScripts/maven.ant.xml
@@ -65,37 +65,61 @@ This buildfile is part of projectlombok.org. It makes maven-compatible repositor
 		<mkdir dir="dist" />
 		
 		<maven.make version-name="${lombok.version}" />
-		<tar destfile="dist/mavenPublish.tar.bz2" compression="bzip2">
-			<tarfileset dir="dist">
+	</target>
+	
+	<target name="maven.publish" depends="maven, -setup.ossrh">
+		<fail>
+			Your lombok clone does not include the OSSRH deployment keys. Contact the core maintainers for these keys;
+			place them in ${gpg.keyrings} to continue.
+			
+			<condition>
+				<not><available file="${gpg.keyrings}" /></not>
+			</condition>
+		</fail>
+		
+		<fail unless="ossrh.password">
+			Your lombok clone does not include the OSSRH (sonatype maven central) password, needed to upload and deploy to maven central. Contact the core maintainers.
+		</fail>
+		
+		<delete quiet="true" dir="build/maven-publish" />
+		<mkdir dir="build/maven-publish" />
+		<copy todir="build/maven-publish">
+			<fileset dir="dist">
 				<include name="lombok-${lombok.version}.jar" />
 				<include name="lombok-${lombok.version}-sources.jar" />
 				<include name="lombok-${lombok.version}-javadoc.jar" />
-			</tarfileset>
-			<tarfileset dir="build" includes="pom.xml" />
-		</tar>
-	</target>
-	
-	<target name="maven.publish" depends="maven, setup.ssh">
-		<ivy:scpUpload
-			from="dist/mavenPublish.tar.bz2"
-			to="/data/lombok/staging"
-			server="projectlombok.org"
-			username="${ssh.username}"
-			keyfile="${ssh.keyfile}"
-			knownHosts="ssh.knownHosts" />
-		<ivy:sshExec
-			cmd="/data/lombok/stagingCmd/publishToMavenCentral"
-			server="projectlombok.org"
-			username="${ssh.username}"
-			keyfile="${ssh.keyfile}"
-			knownHosts="ssh.knownHosts" />
+			</fileset>
+			<fileset dir="build" includes="pom.xml" />
+		</copy>
+		
+		<apply executable="${exe.gpg}" failifexecutionfails="false" resultproperty="gpg.result">
+			<arg value="-ab" />
+			<arg value="--batch" />
+			<arg value="--yes" />
+			<arg value="--homedir" />
+			<arg value="${gpg.keyrings}" />
+			<fileset dir="build/maven-publish" />
+		</apply>
+		
+		<fail>
+			gpg (Gnu Privacy Guard) is not on your path, or ant property exe.gpg is not properly set. Install gpg/add it to your PATH. Alternatively, run with ant -Dexe.gpg=/loc/to/gpg to continue.
+			<condition>
+				<not><isset property="gpg.result" /></not>
+			</condition>
+		</fail>
+		
+		<jar destfile="build/maven-publish/lombok-bundle.jar">
+			<fileset dir="build/maven-publish" />
+		</jar>
+		
+		<exec executable="${exe.curl}" failifexecutionfails="false" resultproperty="curl.result">
+			<arg value="-u" />
+			<arg value="${ossrh.username}:${ossrh.password}" />
+			<arg value="-F" />
+			<arg value="file=@build/maven-publish/lombok-bundle.jar;type=application/java-archive" />
+			<arg value="https://oss.sonatype.org/service/local/staging/bundle_upload" />
+		</exec>
 		<echo>The artifact has been published to staging. Now go to https://oss.sonatype.org/ and log in as Reinier, then doublecheck if all is well and 'release' it.</echo>
-		<ivy:sshExec
-			cmd="/data/lombok/stagingCmd/showMavenCentralPassword"
-			server="projectlombok.org"
-			username="${ssh.username}"
-			keyfile="${ssh.keyfile}"
-			knownHosts="ssh.knownHosts" />
 	</target>
 	
 	<target name="maven.edge" depends="version, dist, javadoc.build" description="Create a maven repo for the current snapshot into a build dir. The intent is for you to put that on a server someplace. Will invoke your local mvn installation.">
@@ -116,12 +140,25 @@ This buildfile is part of projectlombok.org. It makes maven-compatible repositor
 			<arg value="-DpomFile=build/pom.xml" />
 			<arg value="-Durl=file://${basedir}/build/edge-releases" />
 		</exec>
-		
 		<fail>
 			mvn is not on your path and/or MAVEN_HOME is not set. Add mvn to your path or set MAVEN_HOME to continue.
 			<condition>
 				<not><isset property="mvn.result" /></not>
 			</condition>
 		</fail>
+		<copy file="dist/lombok-${lombok.version}.jar" tofile="build/edge-releases/lombok-edge.jar" />
+	</target>
+	
+	<target name="maven.edge.publish" depends="maven.edge, compile.support" description="Creates the maven repo for the snapshot build and publishes it to projectlombok.org">
+		<java classname="lombok.publish.PublishToBucket" failonerror="true">
+			<classpath>
+				<path refid="cp.buildtools" />
+				<pathelement location="build/support" />
+			</classpath>
+			<arg value="${gpg.keyrings}/s3_creds.txt" />
+			<arg value="build/edge-releases" />
+			<arg value="edge" />
+			<arg value="true" />
+		</java>
 	</target>
 </project>
diff --git a/buildScripts/setup.ant.xml b/buildScripts/setup.ant.xml
index 0531a392..ec363cbf 100644
--- a/buildScripts/setup.ant.xml
+++ b/buildScripts/setup.ant.xml
@@ -40,6 +40,10 @@ This buildfile is part of projectlombok.org. It sets up the build itself.
 	<available file="${rtstubs18.loc}" property="rtstubs18.available" />
 	<property name="ssh.configuration.file" location="ssh.configuration" />
 	
+	<property name="gpg.keyrings" location="gpg.keyring" />
+	<property name="exe.gpg" value="gpg" />
+	<property name="exe.curl" value="curl" />
+	
 	<condition property="os.specific.native-swt-lib" value="org.eclipse.swt.gtk.linux.aarch64">
 		<os name="Linux" arch="aarch64" />
 	</condition>
@@ -60,55 +64,6 @@ This buildfile is part of projectlombok.org. It sets up the build itself.
 	</condition>
 	<fail unless="os.specific.native-swt-lib">Full eclipse testing requires downloading a native SWT binding. This script knows how to download for OS = [mac, linux, or windows] and architecture = [aarch64 or x86-64]. You have something different, you unique snowflake you. Your OS: "${os.name}", Your arch: "${os.arch}".</fail>
 	
-	<target name="-setup.ssh.ask">
-		<property file="${ssh.configuration.file}" />
-		<fail>
-			Your ssh.configuration file is corrupted; delete it and rerun this script.
-			<condition>
-				<or>
-					<and>
-						<isset property="ssh.username" />
-						<equals arg1="" arg2="${ssh.username}" />
-					</and>
-					<and>
-						<isset property="ssh.keyfile" />
-						<equals arg1="" arg2="${ssh.keyfile}" />
-					</and>
-				</or>
-			</condition>
-		</fail>
-		<fail>
-			The keyfile configured in your ${ssh.configuration.file} file does not exist.
-			<condition>
-				<and>
-					<isset property="ssh.keyfile" />
-					<not><available file="${ssh.keyfile}" /></not>
-				</and>
-			</condition>
-		</fail>
-		
-		<condition property="ssh.configuration.write">
-			<or>
-				<not><isset property="ssh.username" /></not>
-				<not><isset property="ssh.keyfile" /></not>
-			</or>
-		</condition>
-		<input message="What is your SSH username on the projectlombok.org server? (Enter to abort)." addproperty="ssh.username" />
-		<fail>Aborted<condition><equals arg1="" arg2="${ssh.username}" /></condition></fail>
-		<input message="Where is your ssh keyfile located? (Enter to abort)." addproperty="ssh.keyfile" defaultvalue="${user.home}/.ssh/id_rsa" />
-		<fail>File ${ssh.keyfile} does not exist<condition><not><available file="${ssh.keyfile}" /></not></condition></fail>
-	</target>
-	
-	<target name="-setup.ssh.save" if="ssh.configuration.write">
-		<propertyfile file="${ssh.configuration.file}" comment="SSH connect info for projectlombok.org.">
-			<entry key="ssh.username" value="${ssh.username}" />
-			<entry key="ssh.keyfile" value="${ssh.keyfile}" />
-		</propertyfile>
-		<echo>Your connection info has been written to ${ssh.configuration.file} and will be remembered for future invocations.</echo>
-	</target>
-	
-	<target name="setup.ssh" depends="-setup.ssh.ask, -setup.ssh.save" />
-	
 	<target name="-autoclean.check">
 		<available type="dir" file="build" property="existingbuild.present" />
 		<loadresource property="existingbuild.ver">
@@ -371,6 +326,15 @@ This buildfile is part of projectlombok.org. It sets up the build itself.
 		<ivy:cachepath pathid="cp.test" conf="test" />
 	</target>
 	
+	<target name="-setup.ossrh">
+		<loadfile quiet="true" failonerror="false" encoding="UTF-8" property="ossrh.password" srcfile="${gpg.keyrings}/ossrh.password">
+			<filterchain>
+				<striplinebreaks />
+			</filterchain>
+		</loadfile>
+		<property name="ossrh.username" value="rzwitserloot" />
+	</target>
+	
 	<fail>ant needs to be at least v1.10.0 or higher to build lombok. Your version is: ${ant.version}
 		<condition>
 			<not><antversion atleast="1.10.0" /></not>
diff --git a/buildScripts/website.ant.xml b/buildScripts/website.ant.xml
index d09e230f..aa0d150b 100644
--- a/buildScripts/website.ant.xml
+++ b/buildScripts/website.ant.xml
@@ -136,7 +136,7 @@ such as applying the templates to produce the website, converting the changelog
 		</tar>
 	</target>
 	
-	<target name="website.publish" depends="setup.ssh, website.pack" description="Builds the website, compresses it, sends it to the projectlombok.org server and deploys it">
+	<target name="website.publish" depends="website.pack" description="Builds the website, compresses it, sends it to the projectlombok.org server and deploys it">
 		<ivy:scpUpload
 			from="dist/website.tar.bz2"
 			to="/data/lombok/staging"
@@ -215,7 +215,7 @@ such as applying the templates to produce the website, converting the changelog
 		<echo>File dist/javadoc.tar.bz2 is available</echo>
 	</target>
 	
-	<target name="javadoc.publish" depends="setup.ssh, javadoc.pack">
+	<target name="javadoc.publish" depends="javadoc.pack">
 		<ivy:scpUpload
 			from="dist/javadoc.tar.bz2"
 			to="/data/lombok/staging"
@@ -256,7 +256,7 @@ such as applying the templates to produce the website, converting the changelog
 		</tar>
 	</target>
 	
-	<target name="-release.publish.site" depends="setup.ssh, release.pack">
+	<target name="-release.publish.site" depends="release.pack">
 		<ivy:scpUpload
 			from="dist/website-release.tar.bz2"
 			to="/data/lombok/staging"
@@ -298,7 +298,7 @@ such as applying the templates to produce the website, converting the changelog
 		</tar>
 	</target>
 	
-	<target name="edge.publish" depends="setup.ssh, edge.pack" description="Builds an edge release, sends it to the projectlombok.org server and deploys it by updating the download-edge link">
+	<target name="edge.publish" depends="edge.pack" description="Builds an edge release, sends it to the projectlombok.org server and deploys it by updating the download-edge link">
 		<ivy:scpUpload
 			from="dist/website-edge.tar.bz2"
 			to="/data/lombok/staging"
diff --git a/src/support/lombok/publish/PublishToBucket.java b/src/support/lombok/publish/PublishToBucket.java
new file mode 100644
index 00000000..509a3e91
--- /dev/null
+++ b/src/support/lombok/publish/PublishToBucket.java
@@ -0,0 +1,159 @@
+package lombok.publish;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public class PublishToBucket {
+	private static final boolean DEBUG = false;
+	
+	private static final class AppException extends Exception {
+		AppException(String msg) {
+			super(msg);
+		}
+	}
+	
+	public static void main(String[] args) {
+		try {
+			if (args.length != 4) throw new AppException("4 args required: [path to creds file] [path to file root to upload] [target dir in bucket] [delete files to create a perfect copy or not]");
+			boolean delete;
+			if (args[3].equalsIgnoreCase("true")) delete = true;
+			else if (args[3].equalsIgnoreCase("false")) delete = false;
+			else throw new AppException("4th arg must be 'true' or 'false'");
+			new PublishToBucket().go(args[0], args[1], args[2], delete);
+			System.exit(0);
+		} catch (AppException e) {
+			System.err.println(e.getMessage());
+			System.exit(1);
+		}
+	}
+	
+	private URI endpoint;
+	private String bucketName;
+	private AwsBasicCredentials creds;
+	
+	/**
+	 * @param credsPath path to the creds file; first line in that file is the access key, second line is the secret.
+	 * @param rootPath path to a directory; this directory is replicated into the bucket.
+	 */
+	private void go(String credsPath, String rootPath, String bucketDir, boolean delete) throws AppException {
+		readCreds(Paths.get(credsPath));
+		
+		S3Client s3 = S3Client.builder()
+			.endpointOverride(endpoint)
+			.region(Region.of("auto"))
+			.credentialsProvider(() -> creds)
+			.build();
+		
+		ListObjectsV2Response objList = s3.listObjectsV2(ListObjectsV2Request.builder()
+			.bucket(bucketName)
+			.prefix(bucketDir + "/")
+			.build());
+		
+		Set<String> inBucketBeforeUpload = new HashSet<String>();
+		for (S3Object obj : objList.contents()) inBucketBeforeUpload.add(obj.key());
+		
+		dbg("Already in bucket:\n" +
+			inBucketBeforeUpload.stream().map(x -> "  " + x + "\n").collect(Collectors.joining()) +
+			(inBucketBeforeUpload.isEmpty() ? "  (Nothing)\n" : ""));
+		
+		try {
+			go0(s3, inBucketBeforeUpload, bucketDir + "/", Paths.get(rootPath));
+		} catch (IOException e) {
+			throw new AppException("I/O exception uploading: " + e.getClass() + ": " + e.getMessage());
+		}
+		
+		if (delete) {
+			dbg("Uploads complete. Files to delete:\n" +
+				inBucketBeforeUpload.stream().map(x -> "  " + x + "\n").collect(Collectors.joining()) +
+				(inBucketBeforeUpload.isEmpty() ? "  (Nothing)\n" : ""));
+			
+			if (!inBucketBeforeUpload.isEmpty()) {
+				List<ObjectIdentifier> objsToDelete = new ArrayList<ObjectIdentifier>();
+				for (String key : inBucketBeforeUpload) {
+					objsToDelete.add(ObjectIdentifier.builder().key(key).build());
+				}
+				s3.deleteObjects(DeleteObjectsRequest.builder()
+					.bucket(bucketName)
+					.delete(Delete.builder().objects(objsToDelete).build())
+					.build());
+				dbg("Deletion completed");
+			}
+		}
+	}
+	
+	private static void dbg(String msg) {
+		if (DEBUG) System.out.println(msg);
+	}
+
+	private void go0(S3Client s3, Set<String> inBucketBeforeUpload, String prefix, Path tgt) throws IOException {
+		try (DirectoryStream<Path> ds = Files.newDirectoryStream(tgt)) {
+			for (Path child : ds) {
+				if (Files.isDirectory(child)) {
+					go0(s3, inBucketBeforeUpload, prefix + child.getFileName().toString() + "/", child);
+					continue;
+				}
+				
+				String key = prefix + child.getFileName().toString();
+				PutObjectRequest req = PutObjectRequest.builder()
+					.bucket(bucketName)
+					.key(key)
+					.build();
+				s3.putObject(req, child);
+				boolean overwrote = inBucketBeforeUpload.remove(key);
+				dbg("Uploaded: " + key + (overwrote ? " (overwrote)" : ""));
+			}
+		}
+	}
+	
+	private static final String LINE_DESCRIPTIONS = "accessKey/secretKey/endpoint/bucket";
+	private void readCreds(Path path) throws AppException {
+		try {
+			List<String> lines = Files.readAllLines(path);
+			String accessKey = null, secretKey = null, endPoint = null, bucketName = null;
+			for (String line : lines) {
+				int idx = line.indexOf('#');
+				if (idx != -1) line = line.substring(0, idx);
+				line = line.trim();
+				if (line.isEmpty()) continue;
+				if (accessKey == null) { accessKey = line; continue; }
+				if (secretKey == null) { secretKey = line; continue; }
+				if (endPoint == null) { endPoint = line; continue; }
+				if (bucketName == null) { bucketName = line; continue; }
+				throw new AppException("Too many lines in " + path.toAbsolutePath() + " - only 4 expected: " + LINE_DESCRIPTIONS);
+			}
+			if (bucketName == null) throw new AppException("Expected 3 lines in " + path.toAbsolutePath() + ": " + LINE_DESCRIPTIONS);
+			creds = AwsBasicCredentials.create(accessKey, secretKey);
+			endpoint = URI.create(endPoint);
+			this.bucketName = bucketName;
+		} catch (NoSuchFileException e) {
+			throw new AppException("File with bucket endpoint + credentials is not available. Make file " + path.toAbsolutePath() + "; it should contain something like: \n" +
+				"123456789abcdef0123456789abcdef0  # this is the access key\n" +
+				"123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0  # this is the secret\n" +
+				"https://12345.r2.cloudflarestorage.com   # this is the endpoint\n" +
+				"lombok-data   # this is the bucket name");
+		} catch (IOException e) {
+			throw new AppException("I/O issue reading creds file " + path.toAbsolutePath() + ": " + e.getClass() + ": " + e.getMessage());
+		}
+	}
+}
diff --git a/ssh.knownHosts b/ssh.knownHosts
deleted file mode 100644
index 6cce7f01..00000000
--- a/ssh.knownHosts
+++ /dev/null
@@ -1,2 +0,0 @@
-projectlombok.org:22:ECDSA:X.509:MIIBMzCB7AYHKoZIzj0CATCB4AIBATAsBgcqhkjOPQEBAiEA/////wAAAAEAAAAAAAAAAAAAAAD///////////////8wRAQg/////wAAAAEAAAAAAAAAAAAAAAD///////////////wEIFrGNdiqOpPns+u9VXaYhrxlHQawzFOw9jvOPD4n0mBLBEEEaxfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpZP40Li/hp/m47n60p8D54WK84zV2sxXs7LtkBoN79R9QIhAP////8AAAAA//////////+85vqtpxeehPO5ysL8YyVRAgEBA0IABKwMbAFQuRwz9+PnuBOlc1OqPAYVhg0VBTGQ1G5V6JVfb0CU5GH4NEFp+jEAoGCZNrghB0XLB3d3egfF06ihDgE=
-projectlombok.org:22:EdDSA:X.509:MCowBQYDK2VwAyEA450nCMycc70u7i0qetTrh9yl6+cOS6v4y8clPCnSIHs=