From a2fe1761eade11c81a2bcc52ff8e930556dd9050 Mon Sep 17 00:00:00 2001 From: LexManos Date: Tue, 16 Oct 2018 19:41:14 -0700 Subject: Work attempting to bypass gradle's crappy caching. It caches FAILURES and uses those over the successes we provide in the custom repos! Other work directed twards cleaning up the api, and moved to using maven local which bypasses SOME of the caching and prevents the artifacts from our custom repo from being copied to the gradle central cache. --- .../artifactural/base/artifact/ArtifactBase.java | 9 +- .../base/artifact/SimpleArtifactIdentifier.java | 9 ++ .../base/artifact/SimpleArtifactMetadata.java | 5 + .../base/artifact/StreamableArtifact.java | 5 + .../artifactural/base/cache/ArtifactCacheBase.java | 9 +- .../base/cache/LocatedArtifactCache.java | 43 ++++--- .../artifactural/base/util/HashFunction.java | 112 +++++++++++++++++ .../artifactural/base/util/PatternReplace.java | 140 +++++++++++++++++++++ 8 files changed, 313 insertions(+), 19 deletions(-) create mode 100644 src/shared/java/com/amadornes/artifactural/base/util/HashFunction.java create mode 100644 src/shared/java/com/amadornes/artifactural/base/util/PatternReplace.java (limited to 'src/shared/java') diff --git a/src/shared/java/com/amadornes/artifactural/base/artifact/ArtifactBase.java b/src/shared/java/com/amadornes/artifactural/base/artifact/ArtifactBase.java index e7dd0c2..b164feb 100644 --- a/src/shared/java/com/amadornes/artifactural/base/artifact/ArtifactBase.java +++ b/src/shared/java/com/amadornes/artifactural/base/artifact/ArtifactBase.java @@ -41,8 +41,13 @@ public abstract class ArtifactBase implements Artifact { } @Override - public Artifact.Cached cache(ArtifactCache cache, String specifier) { - return cache.store(this, specifier); + public Artifact.Cached cache(ArtifactCache cache) { + return cache.store(this); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(" + identifier + ", " + type +", " + metadata; } } diff --git a/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactIdentifier.java b/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactIdentifier.java index 231f68b..71f8cc6 100644 --- a/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactIdentifier.java +++ b/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactIdentifier.java @@ -39,4 +39,13 @@ public class SimpleArtifactIdentifier implements ArtifactIdentifier { return extension; } + @Override + public String toString() { + String ret = getGroup() + ':' + getName() + ':' + getVersion(); + if (classifier != null) + ret += ':' + getClassifier(); + if ("jar".equals(extension)) + ret += '@' + getExtension(); + return ret; + } } diff --git a/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactMetadata.java b/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactMetadata.java index 860a655..6dd6195 100644 --- a/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactMetadata.java +++ b/src/shared/java/com/amadornes/artifactural/base/artifact/SimpleArtifactMetadata.java @@ -43,6 +43,11 @@ public class SimpleArtifactMetadata implements ArtifactMetadata { } } + @Override + public String toString() { + return "SimpleArtifactMetadata(" + entries.toString() + ", " + getHash() + ")"; + } + private static class Entry { private final String key, value; diff --git a/src/shared/java/com/amadornes/artifactural/base/artifact/StreamableArtifact.java b/src/shared/java/com/amadornes/artifactural/base/artifact/StreamableArtifact.java index 1b4377a..c0a9f57 100644 --- a/src/shared/java/com/amadornes/artifactural/base/artifact/StreamableArtifact.java +++ b/src/shared/java/com/amadornes/artifactural/base/artifact/StreamableArtifact.java @@ -82,6 +82,11 @@ public class StreamableArtifact extends ArtifactBase { return file; } + @Override + public String toString() { + return "StreamableFileArtifact(" + file + ")"; + } + } } diff --git a/src/shared/java/com/amadornes/artifactural/base/cache/ArtifactCacheBase.java b/src/shared/java/com/amadornes/artifactural/base/cache/ArtifactCacheBase.java index 93ca358..37c9d97 100644 --- a/src/shared/java/com/amadornes/artifactural/base/cache/ArtifactCacheBase.java +++ b/src/shared/java/com/amadornes/artifactural/base/cache/ArtifactCacheBase.java @@ -74,8 +74,8 @@ public abstract class ArtifactCacheBase implements ArtifactCache { } @Override - public Artifact.Cached cache(ArtifactCache cache, String specifier) { - return artifact.cache(cache, specifier); + public Artifact.Cached cache(ArtifactCache cache) { + return artifact.cache(cache); } @Override @@ -100,7 +100,10 @@ public abstract class ArtifactCacheBase implements ArtifactCache { public File getFileLocation() throws MissingArtifactException { return file; } - + @Override + public String toString() { + return "wrapped(" + artifact + ", " + file + ")"; + } }; } diff --git a/src/shared/java/com/amadornes/artifactural/base/cache/LocatedArtifactCache.java b/src/shared/java/com/amadornes/artifactural/base/cache/LocatedArtifactCache.java index 198f240..71fbc9d 100644 --- a/src/shared/java/com/amadornes/artifactural/base/cache/LocatedArtifactCache.java +++ b/src/shared/java/com/amadornes/artifactural/base/cache/LocatedArtifactCache.java @@ -2,11 +2,17 @@ package com.amadornes.artifactural.base.cache; import com.amadornes.artifactural.api.artifact.Artifact; import com.amadornes.artifactural.api.artifact.ArtifactIdentifier; +import com.amadornes.artifactural.base.util.PatternReplace; import java.io.File; +import java.util.AbstractMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class LocatedArtifactCache extends ArtifactCacheBase { - + private static final String PATTERN = "[group]/[name](/[meta_hash])/[version]/[name]-[version](-[classifier])(-[specifier]).[extension]"; private final File path; public LocatedArtifactCache(File path) { @@ -14,22 +20,31 @@ public class LocatedArtifactCache extends ArtifactCacheBase { } @Override - public Artifact.Cached store(Artifact artifact, String specifier) { + public Artifact.Cached store(Artifact artifact) { + return doStore(getPath(artifact), artifact); + } + + public File getPath(Artifact artifact) { ArtifactIdentifier identifier = artifact.getIdentifier(); - File cachePath = new File(path.getAbsolutePath() - .replace("${GROUP}", identifier.getGroup()) - .replace("${NAME}", identifier.getName()) - .replace("${VERSION}", identifier.getVersion()) - .replace("${CLASSIFIER}", identifier.getClassifier()) - .replace("${EXTENSION}", identifier.getExtension()) - .replace("${SPECIFIER}", specifier) - .replace("${META_HASH}", artifact.getMetadata().getHash()) - ); - return doStore(cachePath, artifact); + Map names = Stream.of( + entry("group", identifier.getGroup()), + entry("name", identifier.getName()), + entry("version", identifier.getVersion()), + entry("classifier", identifier.getClassifier()), + entry("extension", identifier.getExtension()), + //entry("specifier", specifier), /? + entry("meta_hash", artifact.getMetadata().getHash()) + ).collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + return new File(path, PatternReplace.replace(PATTERN, names)); } - public static File expand(File path) { - return new File(path, "${GROUP}/${NAME}/${META_HASH}/${NAME}-${VERSION}-${CLASSIFIER}-${SPECIFIER}.${EXTENSION}"); + private static Entry entry(K key, V value) { + return new AbstractMap.SimpleEntry<>(key, value); + } + + @Override + public String toString() { + return "LocatedArtifactCache(" + path + ")"; } } diff --git a/src/shared/java/com/amadornes/artifactural/base/util/HashFunction.java b/src/shared/java/com/amadornes/artifactural/base/util/HashFunction.java new file mode 100644 index 0000000..275b53c --- /dev/null +++ b/src/shared/java/com/amadornes/artifactural/base/util/HashFunction.java @@ -0,0 +1,112 @@ +package com.amadornes.artifactural.base.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; + +import org.apache.commons.io.IOUtils; + +//These are all standard hashing functions the JRE is REQUIRED to have, so add a nice factory that doesnt require catching annoying exceptions; +public enum HashFunction { + MD5("md5", 32), + SHA1("SHA-1", 40), + SHA256("SHA-256", 64); + + private String algo; + private String pad; + + private HashFunction(String algo, int length) { + this.algo = algo; + this.pad = String.format("%0" + length + "d", 0); + } + + public String getExtension() { + return this.name().toLowerCase(Locale.ENGLISH); + } + + public Instance create() { + return new Instance(); + } + + public MessageDigest get() { + try { + return MessageDigest.getInstance(algo); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); //Never happens + } + } + + public String hash(File file) throws IOException { + try (FileInputStream fin = new FileInputStream(file)) { + return hash(fin); + } + } + + public String hash(Iterable files) throws IOException { + MessageDigest hash = get(); + byte[] buf = new byte[1024]; + + for (File file : files) { + if (!file.exists()) + continue; + + try (FileInputStream fin = new FileInputStream(file)) { + int count = -1; + while ((count = fin.read(buf)) != -1) + hash.update(buf, 0, count); + } + } + return pad(new BigInteger(1, hash.digest()).toString(16)); + } + + public String hash(String data) { + return hash(data.getBytes(StandardCharsets.UTF_8)); + } + + public String hash(InputStream stream) throws IOException { + return hash(IOUtils.toByteArray(stream)); + } + + public String hash(byte[] data) { + return pad(new BigInteger(1, get().digest(data)).toString(16)); + } + + public String pad(String hash) { + return (pad + hash).substring(hash.length()); + } + + public class Instance { + private MessageDigest digest = HashFunction.this.get(); + public void update(byte input) { + digest.update(input); + } + public void update(byte[] input) { + digest.update(input); + } + public void update(byte[] input, int offset, int length) { + digest.update(input, offset, length); + } + public void update(ByteBuffer input) { + digest.update(input); + } + public void update(String input) { + update(input.getBytes(StandardCharsets.UTF_8)); + } + public void update(int input) { + update(new byte[] { (byte)((input & 0xFF000000) >> 24), (byte)((input & 0xFF000000) >> 16), (byte)((input & 0xFF000000) >> 8), (byte)((input & 0xFF000000))}); + } + public byte[] digest() { + return digest.digest(); + } + public String finish() { + return pad(new BigInteger(1, digest()).toString(16)); + } + } +} diff --git a/src/shared/java/com/amadornes/artifactural/base/util/PatternReplace.java b/src/shared/java/com/amadornes/artifactural/base/util/PatternReplace.java new file mode 100644 index 0000000..80216d6 --- /dev/null +++ b/src/shared/java/com/amadornes/artifactural/base/util/PatternReplace.java @@ -0,0 +1,140 @@ +package com.amadornes.artifactural.base.util; + +import java.util.Map; + +public class PatternReplace { + /* + * Replaces a patterened string, with support for optional groups. + * Example: + * Values: + * group: net/minecraftforge + * name: forge + * version: 1.0 + * ext: jar + * + * Example: [group]/[name]/[version]/[name]-[version](-[classifier]).[ext] + * {classifier: test} net/minecraftforge/forge/1.0/forge-1.0-test.jar + * {classifier: null} net/minecraftforge/forge/1.0/forge-1.0.jar + * + * Nested Optionals are supported: + * Example: [group]/[name]/[version]/[name]-[version](-[classifier](-[suffix])).[ext] + * {classifier: test, suffix: foo} net/minecraftforge/forge/1.0/forge-1.0-test-foo.jar + * {classifier: test, suffix: foo} net/minecraftforge/forge/1.0/forge-1.0-test.jar + * {classifier: null, suffix: foo} net/minecraftforge/forge/1.0/forge-1.0.jar + * + * Compound optionals are supported: + * Example: [group]/[name]/[version]/[name]-[version](-[classifier]-[suffix]).[ext] + * {classifier: test, suffix: foo} net/minecraftforge/forge/1.0/forge-1.0-test-foo.jar + * {classifier: test, suffix: null} net/minecraftforge/forge/1.0/forge-1.0.jar + * {classifier: null, suffix: foo} net/minecraftforge/forge/1.0/forge-1.0.jar + * + * + * TODO: Support nested names? + * Example: [group]/[name]/[version]/[name]-[version](-[classifier[suffix]]).[ext] + * {classifierFoo: test, suffix: Foo} net/minecraftforge/forge/1.0/forge-1.0-test.jar + * {classifierFoo: null, suffix: Foo} net/minecraftforge/forge/1.0/forge-1.0.jar + */ + public static String replace(String pattern, Map values) { + if (pattern == null) return null; + if (pattern.isEmpty()) return ""; + + Optional optional = null; + StringBuffer name = null; + StringBuffer ret = new StringBuffer(); + + char[] chars = pattern.toCharArray(); + for (int x = 0; x < chars.length; x++) { + char c = chars[x]; + if (c == '\\') { + if (x == chars.length -1) + throw new IllegalArgumentException("Escape character can not be end of pattern: " + pattern); + x++; + ret.append(chars[x]); + continue; + } + switch (c) { + case '[': + if (name != null) + throw new IllegalArgumentException("Nested names are not supported @ " + x + " : " + pattern); + name = new StringBuffer(); + break; + case ']': + if (name == null) + throw new IllegalArgumentException("Name closing found without opening @ " + x + " : " + pattern); + String key = name.toString(); + if (key.isEmpty()) + throw new IllegalArgumentException("Name can not be empty @ " + x + ": " + pattern); + if (optional != null) + optional.setKey(key, values); + else + ret.append(values.get(key)); // appends 'null' if missing, if you want "" then use ([name]) + // Should we have this default to not replacing at all if value is not set to allow chaining? + // Meaning: '[key]' == '[key]' if 'key' is not set. + // Current: '[key]' == 'null' + name = null; + break; + case '(': + optional = new Optional(optional); + break; + case ')': + if (optional == null) + throw new IllegalArgumentException("Optional closing found without opening @ " + x + ": " + pattern); + optional = optional.finish(x, pattern, ret); + break; + default: + if (name != null) + name.append(c); + else if (optional != null) + optional.append(c); + else + ret.append(c); + } + } + if (optional != null) + throw new IllegalArgumentException("Missing closing of optional value: " + pattern); + if (name != null) + throw new IllegalArgumentException("Missing closing of name entry: " + pattern); + return ret.toString(); + } + + public static String quote(String value) { + return value.replaceAll("\\", "\\\\") + .replaceAll("(", "\\(") + .replaceAll(")", "\\)") + .replaceAll("[", "\\[") + .replaceAll("]", "\\]"); + } + + private static class Optional { + private final Optional parent; + private final StringBuffer buf = new StringBuffer(); + private boolean hadAll = true; + private boolean hadValue = false; + + private Optional(Optional parent) { + this.parent = parent; + } + + public void append(char c) { + buf.append(c); + } + + private void setKey(String key, Map values) { + hadValue = true; + String value = values.get(key); + if (value != null && !value.isEmpty()) { + hadAll &= true; + buf.append(value); + } else + hadAll = false; + } + + public Optional finish(int position, String pattern, StringBuffer ret) { + if (!hadValue) + throw new IllegalArgumentException("Invalid optional, missing inner name @ " + position +": " + pattern); + if (hadAll) + (parent == null ? ret : parent.buf).append(buf); + return parent; + } + } +} -- cgit