From a2fe1761eade11c81a2bcc52ff8e930556dd9050 Mon Sep 17 00:00:00 2001 From: LexManos Date: Tue, 16 Oct 2018 19:41:14 -0700 Subject: Work attempting to bypass gradle's crappy caching. It caches FAILURES and uses those over the successes we provide in the custom repos! Other work directed twards cleaning up the api, and moved to using maven local which bypasses SOME of the caching and prevents the artifacts from our custom repo from being copied to the gradle central cache. --- .../gradle/GradleRepositoryAdapter.java | 411 +++++++++------------ .../artifactural/gradle/ReflectionUtils.java | 49 ++- 2 files changed, 213 insertions(+), 247 deletions(-) (limited to 'src/gradlecomp') diff --git a/src/gradlecomp/java/com/amadornes/artifactural/gradle/GradleRepositoryAdapter.java b/src/gradlecomp/java/com/amadornes/artifactural/gradle/GradleRepositoryAdapter.java index 4bac887..2418ea9 100644 --- a/src/gradlecomp/java/com/amadornes/artifactural/gradle/GradleRepositoryAdapter.java +++ b/src/gradlecomp/java/com/amadornes/artifactural/gradle/GradleRepositoryAdapter.java @@ -2,108 +2,181 @@ package com.amadornes.artifactural.gradle; import com.amadornes.artifactural.api.artifact.Artifact; import com.amadornes.artifactural.api.artifact.ArtifactIdentifier; +import com.amadornes.artifactural.api.artifact.MissingArtifactException; import com.amadornes.artifactural.api.repository.Repository; import com.amadornes.artifactural.base.artifact.SimpleArtifactIdentifier; -import com.google.common.io.CountingInputStream; -import org.apache.commons.io.IOUtils; -import org.gradle.api.Action; -import org.gradle.api.NamedDomainObjectCollection; -import org.gradle.api.Transformer; +import com.amadornes.artifactural.base.cache.LocatedArtifactCache; + +import org.gradle.api.artifacts.ComponentMetadataSupplierDetails; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.artifacts.component.ComponentArtifactIdentifier; +import org.gradle.api.artifacts.component.ModuleComponentIdentifier; import org.gradle.api.artifacts.dsl.RepositoryHandler; -import org.gradle.api.artifacts.repositories.ArtifactRepository; -import org.gradle.api.artifacts.repositories.MavenArtifactRepository; +import org.gradle.api.attributes.AttributeContainer; +import org.gradle.api.internal.artifacts.BaseRepositoryFactory; import org.gradle.api.internal.artifacts.ivyservice.ivyresolve.ConfiguredModuleComponentRepository; +import org.gradle.api.internal.artifacts.ivyservice.ivyresolve.ModuleComponentRepositoryAccess; +import org.gradle.api.internal.artifacts.ivyservice.resolveengine.artifact.ResolvableArtifact; import org.gradle.api.internal.artifacts.repositories.AbstractArtifactRepository; -import org.gradle.api.internal.artifacts.repositories.DefaultMavenArtifactRepository; +import org.gradle.api.internal.artifacts.repositories.DefaultMavenLocalArtifactRepository; import org.gradle.api.internal.artifacts.repositories.ResolutionAwareRepository; -import org.gradle.api.internal.artifacts.repositories.resolver.ExternalResourceArtifactResolver; -import org.gradle.api.internal.artifacts.repositories.resolver.ExternalResourceResolver; import org.gradle.api.internal.artifacts.repositories.resolver.MavenResolver; -import org.gradle.api.resources.ResourceException; -import org.gradle.internal.resource.AbstractExternalResource; -import org.gradle.internal.resource.ExternalResource; +import org.gradle.api.internal.artifacts.repositories.resolver.MetadataFetchingCost; +import org.gradle.api.internal.attributes.AttributesSchemaInternal; +import org.gradle.api.internal.attributes.ImmutableAttributesFactory; +import org.gradle.api.internal.component.ArtifactType; +import org.gradle.internal.action.InstantiatingAction; +import org.gradle.internal.component.external.model.ComponentVariant; +import org.gradle.internal.component.external.model.ModuleComponentArtifactMetadata; +import org.gradle.internal.component.external.model.ModuleComponentResolveMetadata; +import org.gradle.internal.component.external.model.ModuleDependencyMetadata; +import org.gradle.internal.component.external.model.MutableModuleComponentResolveMetadata; +import org.gradle.internal.component.model.ComponentArtifactMetadata; +import org.gradle.internal.component.model.ComponentOverrideMetadata; +import org.gradle.internal.component.model.ComponentResolveMetadata; +import org.gradle.internal.component.model.ConfigurationMetadata; +import org.gradle.internal.component.model.ModuleSource; +import org.gradle.internal.hash.HashValue; +import org.gradle.internal.impldep.com.google.common.base.Optional; +import org.gradle.internal.impldep.com.google.common.collect.ImmutableList; +import org.gradle.internal.nativeintegration.filesystem.FileSystem; +import org.gradle.internal.nativeintegration.services.FileSystems; +import org.gradle.internal.resolve.result.BuildableArtifactResolveResult; +import org.gradle.internal.resolve.result.BuildableArtifactSetResolveResult; +import org.gradle.internal.resolve.result.BuildableComponentArtifactsResolveResult; +import org.gradle.internal.resolve.result.BuildableModuleComponentMetaDataResolveResult; +import org.gradle.internal.resolve.result.BuildableModuleVersionListingResolveResult; import org.gradle.internal.resource.ExternalResourceName; -import org.gradle.internal.resource.ExternalResourceReadResult; import org.gradle.internal.resource.ExternalResourceRepository; -import org.gradle.internal.resource.ExternalResourceWriteResult; -import org.gradle.internal.resource.ReadableContent; -import org.gradle.internal.resource.ResourceExceptions; -import org.gradle.internal.resource.metadata.DefaultExternalResourceMetaData; +import org.gradle.internal.resource.LocalBinaryResource; +import org.gradle.internal.resource.local.FileResourceRepository; +import org.gradle.internal.resource.local.LocalFileStandInExternalResource; +import org.gradle.internal.resource.local.LocallyAvailableExternalResource; import org.gradle.internal.resource.metadata.ExternalResourceMetaData; +import org.gradle.internal.resource.transfer.DefaultCacheAwareExternalResourceAccessor; -import javax.annotation.Nullable; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; public class GradleRepositoryAdapter extends AbstractArtifactRepository implements ResolutionAwareRepository { private static final Pattern URL_PATTERN = Pattern.compile( - "^/(?\\S+(?:/\\S+)*)/(?\\S+)/(?\\S+)/" + + "^(?\\S+(?:/\\S+)*)/(?\\S+)/(?\\S+)/" + "\\2-\\3(?:-(?[^.\\s]+))?\\.(?\\S+)$"); - public static GradleRepositoryAdapter add(RepositoryHandler handler, String name, Object url, Repository repository) { - // Create the real maven test we'll be using and remove it - MavenArtifactRepository maven = handler.maven($ -> { - $.setName(name); - $.setUrl(url); - }); - handler.remove(maven); + public static GradleRepositoryAdapter add(RepositoryHandler handler, String name, File local, Repository repository) { + BaseRepositoryFactory factory = ReflectionUtils.get(handler, "repositoryFactory"); // We reflect here and create it manually so it DOESN'T get attached. + DefaultMavenLocalArtifactRepository maven = (DefaultMavenLocalArtifactRepository)factory.createMavenLocalRepository(); // We use maven local because it bypasses the caching and coping to .m2 + maven.setUrl(local); + maven.setName(name); - // Add our own custom test instead, using the real one in the background - GradleRepositoryAdapter repo = new GradleRepositoryAdapter((DefaultMavenArtifactRepository) maven, repository); + GradleRepositoryAdapter repo = new GradleRepositoryAdapter(repository, maven); + repo.setName(name); handler.add(repo); return repo; } - private final DefaultMavenArtifactRepository maven; private final Repository repository; + private final DefaultMavenLocalArtifactRepository local; - private GradleRepositoryAdapter(DefaultMavenArtifactRepository maven, Repository repository) { - this.maven = maven; + private GradleRepositoryAdapter(Repository repository, DefaultMavenLocalArtifactRepository local) { this.repository = repository; - } - - @Override - public String getName() { - return maven.getName(); // Proxy to the real repo - } - - @Override - public void setName(String name) { - maven.setName(name); // Proxy to the real repo + this.local = local; } @Override public String getDisplayName() { - return maven.getDisplayName(); // Proxy to the real repo - } - - @Override - public void onAddToContainer(NamedDomainObjectCollection container) { - // No-op. The real repo will get this already + return local.getDisplayName(); } @Override public ConfiguredModuleComponentRepository createResolver() { - MavenResolver resolver = (MavenResolver) maven.createResolver(); - ExternalResourceRepository repo = new StreamingRepo(); - - ExternalResourceArtifactResolver artifactResolver = ReflectionUtils.invoke(resolver, ExternalResourceResolver.class, "createArtifactResolver"); - ReflectionUtils.alter(resolver, "repository", prev -> repo); - ReflectionUtils.alter(resolver, "mavenMetaDataLoader.cacheAwareExternalResourceAccessor.delegate", prev -> repo); - ReflectionUtils.alter(artifactResolver, "repository", prev -> repo); + MavenResolver resolver = (MavenResolver)local.createResolver(); + + GeneratingFileResourceRepository repo = new GeneratingFileResourceRepository(); + ReflectionUtils.alter(resolver, "repository", prev -> repo); // ExternalResourceResolver.repository + //ReflectionUtils.alter(resolver, "metadataSources", ); //ExternalResourceResolver.metadataSources We need to fix these from returning 'missing' + // MavenResolver -> MavenMetadataLoader -> FileCacheAwareExternalResourceAccessor -> DefaultCacheAwareExternalResourceAccessor + DefaultCacheAwareExternalResourceAccessor accessor = ReflectionUtils.get(resolver, "mavenMetaDataLoader.cacheAwareExternalResourceAccessor.delegate"); + ReflectionUtils.alter(accessor, "delegate", prev -> repo); // DefaultCacheAwareExternalResourceAccessor.delegate + ReflectionUtils.alter(accessor, "fileResourceRepository", prev -> repo); // DefaultCacheAwareExternalResourceAccessor.fileResourceRepository + //ReflectionUtils.alter(resolver, "localAccess", AccessWrapper::new); + + //return resolver; + //return new Resolver(resolver); + return new ConfiguredModuleComponentRepository() { + private final ModuleComponentRepositoryAccess local = wrap(resolver.getLocalAccess()); + private final ModuleComponentRepositoryAccess remote = wrap(resolver.getRemoteAccess()); + @Override public String getId() { return resolver.getId(); } + @Override public String getName() { return resolver.getName(); } + @Override public ModuleComponentRepositoryAccess getLocalAccess() { return local; } + @Override public ModuleComponentRepositoryAccess getRemoteAccess() { return remote; } + @Override public Map getArtifactCache() { return resolver.getArtifactCache(); } + @Override public InstantiatingAction getComponentMetadataSupplier() { return resolver.getComponentMetadataSupplier(); } + @Override public boolean isDynamicResolveMode() { return resolver.isDynamicResolveMode(); } + @Override public boolean isLocal() { return resolver.isLocal(); } + + private ModuleComponentRepositoryAccess wrap(ModuleComponentRepositoryAccess delegate) { + return new ModuleComponentRepositoryAccess() { + @Override + public void resolveComponentMetaData(ModuleComponentIdentifier moduleComponentIdentifier, ComponentOverrideMetadata requestMetaData, BuildableModuleComponentMetaDataResolveResult result) { + delegate.resolveComponentMetaData(moduleComponentIdentifier, requestMetaData, result); + if (result.getState() == BuildableModuleComponentMetaDataResolveResult.State.Resolved) { + ModuleComponentResolveMetadata meta = result.getMetaData(); + if (meta.isMissing()) { + MutableModuleComponentResolveMetadata mutable = meta.asMutable(); + mutable.setChanging(true); + mutable.setMissing(false); + result.resolved(mutable.asImmutable()); + } + } + } + + @Override + public void listModuleVersions(ModuleDependencyMetadata dependency, BuildableModuleVersionListingResolveResult result) { + delegate.listModuleVersions(dependency, result); + } + + @Override + public void resolveArtifacts(ComponentResolveMetadata component, BuildableComponentArtifactsResolveResult result) { + delegate.resolveArtifacts(component, result); + } + + @Override + public void resolveArtifactsWithType(ComponentResolveMetadata component, ArtifactType artifactType, BuildableArtifactSetResolveResult result) { + delegate.resolveArtifactsWithType(component, artifactType, result); + } + + @Override + public void resolveArtifact(ComponentArtifactMetadata artifact, ModuleSource moduleSource, BuildableArtifactResolveResult result) { + delegate.resolveArtifact(artifact, moduleSource, result); + } + + @Override + public MetadataFetchingCost estimateMetadataFetchingCost(ModuleComponentIdentifier moduleComponentIdentifier) { + return delegate.estimateMetadataFetchingCost(moduleComponentIdentifier); + } + }; + } + }; + } - return resolver; + private static String cleanRoot(URI uri) { + String ret = uri.normalize().getPath().replace('\\', '/'); + if (!ret.endsWith("/")) ret += '/'; + return ret; } - private class StreamingRepo implements ExternalResourceRepository { + private class GeneratingFileResourceRepository implements FileResourceRepository { + private final FileSystem fileSystem = FileSystems.getDefault(); + private final String root = cleanRoot(GradleRepositoryAdapter.this.local.getUrl()); + private final LocatedArtifactCache cache = new LocatedArtifactCache(new File(root)); @Override public ExternalResourceRepository withProgressLogging() { @@ -111,204 +184,66 @@ public class GradleRepositoryAdapter extends AbstractArtifactRepository implemen } @Override - public ExternalResource resource(ExternalResourceName name, boolean revalidate) { - URI uri = name.getUri(); - Matcher matcher = URL_PATTERN.matcher(uri.getPath()); - if (!matcher.matches()) return new NullExternalResource(uri); - ArtifactIdentifier identifier = new SimpleArtifactIdentifier( - matcher.group("group").replace('/', '.'), - matcher.group("name"), - matcher.group("version"), - matcher.group("classifier"), - matcher.group("extension")); - Artifact artifact = repository.getArtifact(identifier); - if (!artifact.isPresent()) return new NullExternalResource(uri); - return new CustomArtifactExternalResource(uri, artifact); + public LocalBinaryResource localResource(File file) { + System.out.println("localResource: " + file); + return null; } @Override - public ExternalResource resource(ExternalResourceName name) { - return resource(name, false); - } - - } - - private class CustomArtifactExternalResource extends AbstractExternalResource { - - private final URI uri; - private final Artifact artifact; - - private CustomArtifactExternalResource(URI uri, Artifact artifact) { - this.uri = uri; - this.artifact = artifact; + public LocallyAvailableExternalResource resource(File file) { + System.out.println("resource(File): " + file); + return findArtifact(file.getAbsolutePath().replace('\\', '/')); } @Override - public String getDisplayName() { - return uri.toString(); + public LocallyAvailableExternalResource resource(ExternalResourceName location) { + return resource(location, false); } @Override - public URI getURI() { - return uri; + public LocallyAvailableExternalResource resource(ExternalResourceName location, boolean revalidate) { + System.out.println("resource(ExternalResourceName,boolean): " + location + ", " + revalidate); + return findArtifact(location.getUri().getPath().replace('\\', '/')); } - @Nullable @Override - public ExternalResourceReadResult writeToIfPresent(File file) { - try { - if (!artifact.isPresent()) return null; - FileOutputStream out = new FileOutputStream(file); - ExternalResourceReadResult result = writeTo(out); - out.close(); - return result; - } catch (IOException ex) { - return null; - } + public LocallyAvailableExternalResource resource(File file, URI originUri, ExternalResourceMetaData originMetadata) { + System.out.println("resource(File,URI,ExternalResourceMetaData): " + file + ", " + originUri + ", " + originMetadata); + return findArtifact(file.getAbsolutePath().replace('\\', '/')); } - @Override - public ExternalResourceReadResult writeTo(OutputStream out) throws ResourceException { - return withContent(in -> { - try { - IOUtils.copy(in, out); - } catch (IOException ex) { - throw ResourceExceptions.failure(uri, "Failed to write resource!", ex); + private LocallyAvailableExternalResource findArtifact(String path) { + if (path.startsWith(root)) { + String relative = path.substring(root.length()); + System.out.println(" Relative: " + relative); + Matcher matcher = URL_PATTERN.matcher(relative); + if (!matcher.matches()) { + System.out.println(" Matcher Failed: " + relative); + } else { + ArtifactIdentifier identifier = new SimpleArtifactIdentifier( + matcher.group("group").replace('/', '.'), + matcher.group("name"), + matcher.group("version"), + matcher.group("classifier"), + matcher.group("extension")); + Artifact artifact = repository.getArtifact(identifier); + return wrap(artifact, identifier); } - }); - } - - @Override - public ExternalResourceReadResult withContent(Action action) throws ResourceException { - try { - if (!artifact.isPresent()) throw ResourceExceptions.getMissing(uri); - CountingInputStream in = new CountingInputStream(artifact.openStream()); - action.execute(in); - in.close(); - return ExternalResourceReadResult.of(in.getCount()); - } catch (IOException ex) { - throw ResourceExceptions.failure(uri, "Failed to write resource!", ex); - } - } - - @Nullable - @Override - public ExternalResourceReadResult withContentIfPresent(Transformer transformer) { - try { - if (!artifact.isPresent()) return null; - CountingInputStream in = new CountingInputStream(artifact.openStream()); - T result = transformer.transform(in); - in.close(); - return ExternalResourceReadResult.of(in.getCount(), result); - } catch (IOException ex) { - return null; - } - } - - @Nullable - @Override - public ExternalResourceReadResult withContentIfPresent(ContentAction contentAction) { - try { - if (!artifact.isPresent()) return null; - CountingInputStream in = new CountingInputStream(artifact.openStream()); - T result = contentAction.execute(in, getMetaData()); - in.close(); - return ExternalResourceReadResult.of(in.getCount(), result); - } catch (IOException ex) { - return null; + } else { + System.out.println("Unknown root: " + path); } + return new LocalFileStandInExternalResource(new File(path), fileSystem); } - @Override - public ExternalResourceWriteResult put(ReadableContent readableContent) throws ResourceException { - throw ResourceExceptions.putFailed(uri, null); - } - - @Nullable - @Override - public List list() throws ResourceException { - return null; - } - - @Nullable - @Override - public ExternalResourceMetaData getMetaData() { + private LocallyAvailableExternalResource wrap(Artifact artifact, ArtifactIdentifier id) { + if (!artifact.isPresent()) + return new LocalFileStandInExternalResource(cache.getPath(artifact), fileSystem); + Artifact.Cached cached = artifact.optionallyCache(cache); try { - if (!artifact.isPresent()) return null; - InputStream stream = artifact.openStream(); - int length = stream.available(); - stream.close(); - return new DefaultExternalResourceMetaData(uri, 0, length); - } catch (IOException ex) { - return null; + return new LocalFileStandInExternalResource(cached.asFile(), fileSystem); + } catch (MissingArtifactException | IOException e) { + throw new RuntimeException(e); } } - - } - - private class NullExternalResource extends AbstractExternalResource { - - private final URI uri; - - private NullExternalResource(URI uri) { - this.uri = uri; - } - - @Override - public String getDisplayName() { - return uri.toString(); - } - - @Override - public URI getURI() { - return uri; - } - - @Nullable - @Override - public ExternalResourceReadResult writeToIfPresent(File destination) throws ResourceException { - return null; - } - - @Override - public ExternalResourceReadResult writeTo(OutputStream destination) throws ResourceException { - throw ResourceExceptions.getMissing(uri); - } - - @Override - public ExternalResourceReadResult withContent(Action readAction) throws ResourceException { - throw ResourceExceptions.getMissing(uri); - } - - @Nullable - @Override - public ExternalResourceReadResult withContentIfPresent(Transformer readAction) { - return null; - } - - @Nullable - @Override - public ExternalResourceReadResult withContentIfPresent(ContentAction readAction) { - return null; - } - - @Override - public ExternalResourceWriteResult put(ReadableContent source) throws ResourceException { - throw ResourceExceptions.getMissing(uri); - } - - @Nullable - @Override - public List list() { - return null; - } - - @Nullable - @Override - public ExternalResourceMetaData getMetaData() { - return null; - } - } - } diff --git a/src/gradlecomp/java/com/amadornes/artifactural/gradle/ReflectionUtils.java b/src/gradlecomp/java/com/amadornes/artifactural/gradle/ReflectionUtils.java index 5dcb9a3..f43b33c 100644 --- a/src/gradlecomp/java/com/amadornes/artifactural/gradle/ReflectionUtils.java +++ b/src/gradlecomp/java/com/amadornes/artifactural/gradle/ReflectionUtils.java @@ -6,28 +6,42 @@ import java.util.function.UnaryOperator; public class ReflectionUtils { + @SuppressWarnings("unchecked") public static void alter(Object target, String name, UnaryOperator operator) { try { - Object prev = target; - Field f = null; - for (String n : name.split("\\.")) { - f = findField(target.getClass(), n); - if (f == null) throw new IllegalStateException("Could not find '" + name + "'"); - f.setAccessible(true); - prev = target; - target = f.get(target); + int idx = name.lastIndexOf('.'); + if (idx != -1) { + target = drillField(target, name.substring(0, idx)); + if (target == null) throw new IllegalStateException("Could not find field '" + name + "'"); + name = name.substring(idx + 1); } + Field f = findField(target.getClass(), name); if (f == null) throw new IllegalStateException("Could not find '" + name + "'"); - f.set(prev, operator.apply((T) target)); + f.set(target, operator.apply((T)f.get(target))); } catch (IllegalAccessException ex) { throw new RuntimeException(ex); } } + private static Object drillField(Object obj, String path) { + for (String name : path.split("\\.")) { + if (obj == null) return null; + Field f = findField(obj.getClass(), name); + if (f == null) return null; + try { + obj = f.get(obj); + } catch (IllegalAccessException e) { + return null; + } + } + return obj; + } + private static Field findField(Class clazz, String name) { while (clazz != Object.class) { for (Field f : clazz.getDeclaredFields()) { if (f.getName().equals(name)) { + f.setAccessible(true); return f; } } @@ -39,6 +53,7 @@ public class ReflectionUtils { /** * Invokes a method (can be private). */ + @SuppressWarnings("unchecked") public static T invoke(Object target, Class type, String name, Object... args) { try { Method method = type.getDeclaredMethod(name); @@ -49,4 +64,20 @@ public class ReflectionUtils { } } + @SuppressWarnings("unchecked") + public static T get(Object target, String name) { + try { + int idx = name.lastIndexOf('.'); + if (idx != -1) { + target = drillField(target, name.substring(0, idx)); + if (target == null) throw new IllegalStateException("Could not find field '" + name + "'"); + name = name.substring(idx + 1); + } + Field f = findField(target.getClass(), name); + if (f == null) throw new IllegalStateException("Could not find '" + name + "'"); + return (T)f.get(target); + } catch (IllegalAccessException ex) { + throw new RuntimeException(ex); + } + } } -- cgit