From fed6a350a80f6daa8c170770e5f300a0d5aa0894 Mon Sep 17 00:00:00 2001 From: Luck Date: Sun, 26 Jun 2022 18:59:02 +0100 Subject: Include info about number of ticks in a profile --- .../spark/common/sampler/AbstractSampler.java | 36 +++++++++++++++++++--- .../me/lucko/spark/common/sampler/Sampler.java | 2 +- .../lucko/spark/common/sampler/SamplerBuilder.java | 6 ++-- .../spark/common/sampler/async/AsyncSampler.java | 10 +++--- .../spark/common/sampler/java/JavaSampler.java | 12 ++++---- .../common/sampler/java/TickedDataAggregator.java | 15 +++++++++ 6 files changed, 61 insertions(+), 20 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index ce466a0..1c217db 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -27,6 +27,7 @@ import me.lucko.spark.common.platform.serverconfig.ServerConfigProvider; import me.lucko.spark.common.sampler.aggregator.DataAggregator; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; +import me.lucko.spark.common.tick.TickHook; import me.lucko.spark.common.util.ClassSourceLookup; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; @@ -41,6 +42,9 @@ import java.util.concurrent.CompletableFuture; */ public abstract class AbstractSampler implements Sampler { + /** The spark platform instance */ + protected final SparkPlatform platform; + /** The interval to wait between sampling, in microseconds */ protected final int interval; @@ -50,8 +54,11 @@ public abstract class AbstractSampler implements Sampler { /** The time when sampling first began */ protected long startTime = -1; + /** The game tick when sampling first began */ + protected int startTick = -1; + /** The unix timestamp (in millis) when this sampler should automatically complete. */ - protected final long endTime; // -1 for nothing + protected final long autoEndTime; // -1 for nothing /** A future to encapsulate the completion of this sampler instance */ protected final CompletableFuture future = new CompletableFuture<>(); @@ -59,10 +66,11 @@ public abstract class AbstractSampler implements Sampler { /** The garbage collector statistics when profiling started */ protected Map initialGcStats; - protected AbstractSampler(int interval, ThreadDumper threadDumper, long endTime) { + protected AbstractSampler(SparkPlatform platform, int interval, ThreadDumper threadDumper, long autoEndTime) { + this.platform = platform; this.interval = interval; this.threadDumper = threadDumper; - this.endTime = endTime; + this.autoEndTime = autoEndTime; } @Override @@ -74,8 +82,8 @@ public abstract class AbstractSampler implements Sampler { } @Override - public long getEndTime() { - return this.endTime; + public long getAutoEndTime() { + return this.autoEndTime; } @Override @@ -91,6 +99,16 @@ public abstract class AbstractSampler implements Sampler { return this.initialGcStats; } + @Override + public void start() { + this.startTime = System.currentTimeMillis(); + + TickHook tickHook = this.platform.getTickHook(); + if (tickHook != null) { + this.startTick = tickHook.getCurrentTick(); + } + } + protected void writeMetadataToProto(SamplerData.Builder proto, SparkPlatform platform, CommandSender creator, String comment, DataAggregator dataAggregator) { SamplerMetadata.Builder metadata = SamplerMetadata.newBuilder() .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) @@ -105,6 +123,14 @@ public abstract class AbstractSampler implements Sampler { metadata.setComment(comment); } + if (this.startTick != -1) { + TickHook tickHook = this.platform.getTickHook(); + if (tickHook != null) { + int numberOfTicks = tickHook.getCurrentTick() - this.startTick; + metadata.setNumberOfTicks(numberOfTicks); + } + } + try { metadata.setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(getInitialGcStats())); } catch (Exception e) { diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java index 845043f..84f2da1 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java @@ -57,7 +57,7 @@ public interface Sampler { * * @return the end time, or -1 if undefined */ - long getEndTime(); + long getAutoEndTime(); /** * Gets a future to encapsulate the completion of the sampler diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java index 88cf018..88b9d91 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java @@ -97,11 +97,11 @@ public class SamplerBuilder { Sampler sampler; if (this.ticksOver != -1 && this.tickHook != null) { - sampler = new JavaSampler(intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative, this.tickHook, this.ticksOver); + sampler = new JavaSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative, this.tickHook, this.ticksOver); } else if (this.useAsyncProfiler && !(this.threadDumper instanceof ThreadDumper.Regex) && AsyncProfilerAccess.INSTANCE.checkSupported(platform)) { - sampler = new AsyncSampler(intervalMicros, this.threadDumper, this.threadGrouper, this.timeout); + sampler = new AsyncSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout); } else { - sampler = new JavaSampler(intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative); + sampler = new JavaSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative); } sampler.start(); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index 5cb7fdc..d8288da 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -65,8 +65,8 @@ public class AsyncSampler extends AbstractSampler { /** The executor used for timeouts */ private ScheduledExecutorService timeoutExecutor; - public AsyncSampler(int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime) { - super(interval, threadDumper, endTime); + public AsyncSampler(SparkPlatform platform, int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime) { + super(platform, interval, threadDumper, endTime); this.profiler = AsyncProfilerAccess.INSTANCE.getProfiler(); this.dataAggregator = new AsyncDataAggregator(threadGrouper); } @@ -90,7 +90,7 @@ public class AsyncSampler extends AbstractSampler { */ @Override public void start() { - this.startTime = System.currentTimeMillis(); + super.start(); try { this.outputFile = TemporaryFiles.create("spark-profile-", ".jfr.tmp"); @@ -120,11 +120,11 @@ public class AsyncSampler extends AbstractSampler { } private void scheduleTimeout() { - if (this.endTime == -1) { + if (this.autoEndTime == -1) { return; } - long delay = this.endTime - System.currentTimeMillis(); + long delay = this.autoEndTime - System.currentTimeMillis(); if (delay <= 0) { return; } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java index cfa0a0f..913faee 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java @@ -63,19 +63,19 @@ public class JavaSampler extends AbstractSampler implements Runnable { /** Responsible for aggregating and then outputting collected sampling data */ private final JavaDataAggregator dataAggregator; - public JavaSampler(int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime, boolean ignoreSleeping, boolean ignoreNative) { - super(interval, threadDumper, endTime); + public JavaSampler(SparkPlatform platform, int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime, boolean ignoreSleeping, boolean ignoreNative) { + super(platform, interval, threadDumper, endTime); this.dataAggregator = new SimpleDataAggregator(this.workerPool, threadGrouper, interval, ignoreSleeping, ignoreNative); } - public JavaSampler(int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime, boolean ignoreSleeping, boolean ignoreNative, TickHook tickHook, int tickLengthThreshold) { - super(interval, threadDumper, endTime); + public JavaSampler(SparkPlatform platform, int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime, boolean ignoreSleeping, boolean ignoreNative, TickHook tickHook, int tickLengthThreshold) { + super(platform, interval, threadDumper, endTime); this.dataAggregator = new TickedDataAggregator(this.workerPool, threadGrouper, interval, ignoreSleeping, ignoreNative, tickHook, tickLengthThreshold); } @Override public void start() { - this.startTime = System.currentTimeMillis(); + super.start(); this.task = this.workerPool.scheduleAtFixedRate(this, 0, this.interval, TimeUnit.MICROSECONDS); } @@ -89,7 +89,7 @@ public class JavaSampler extends AbstractSampler implements Runnable { // this is effectively synchronized, the worker pool will not allow this task // to concurrently execute. try { - if (this.endTime != -1 && this.endTime <= System.currentTimeMillis()) { + if (this.autoEndTime != -1 && this.autoEndTime <= System.currentTimeMillis()) { this.future.complete(this); stop(); return; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java index e817828..e062f31 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Implementation of {@link DataAggregator} which supports only including sampling data from "ticks" @@ -47,6 +48,9 @@ public class TickedDataAggregator extends JavaDataAggregator { /** The expected number of samples in each tick */ private final int expectedSize; + /** The number of ticks aggregated so far */ + private final AtomicInteger numberOfTicks = new AtomicInteger(); + private final Object mutex = new Object(); // state @@ -64,10 +68,16 @@ public class TickedDataAggregator extends JavaDataAggregator { @Override public SamplerMetadata.DataAggregator getMetadata() { + // push the current tick (so numberOfTicks is accurate) + synchronized (this.mutex) { + pushCurrentTick(); + } + return SamplerMetadata.DataAggregator.newBuilder() .setType(SamplerMetadata.DataAggregator.Type.TICKED) .setThreadGrouper(this.threadGrouper.asProto()) .setTickLengthThreshold(this.tickLengthThreshold) + .setNumberOfIncludedTicks(this.numberOfTicks.get()) .build(); } @@ -97,6 +107,7 @@ public class TickedDataAggregator extends JavaDataAggregator { return; } + this.numberOfTicks.incrementAndGet(); this.workerPool.submit(currentData); } @@ -110,6 +121,10 @@ public class TickedDataAggregator extends JavaDataAggregator { return super.exportData(); } + public int getNumberOfTicks() { + return this.numberOfTicks.get(); + } + private final class TickList implements Runnable { private final List list; -- cgit From 73dd214ae66bab483ee8b4f0ed03881466da92e8 Mon Sep 17 00:00:00 2001 From: Luck Date: Tue, 12 Jul 2022 22:37:59 +0100 Subject: Improve/fix game thread dumper --- .../main/java/me/lucko/spark/common/sampler/ThreadDumper.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java index 9d54f50..fe3a6a7 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java @@ -83,10 +83,8 @@ public interface ThreadDumper { return Objects.requireNonNull(this.dumper, "dumper"); } - public void ensureSetup() { - if (this.dumper == null) { - this.dumper = new Specific(new long[]{Thread.currentThread().getId()}); - } + public void setThread(Thread thread) { + this.dumper = new Specific(new long[]{thread.getId()}); } } @@ -98,6 +96,10 @@ public interface ThreadDumper { private Set threads; private Set threadNamesLowerCase; + public Specific(Thread thread) { + this.ids = new long[]{thread.getId()}; + } + public Specific(long[] ids) { this.ids = ids; } -- cgit From 06de991f44f3f0f33eed21fb92224a395a2a92ff Mon Sep 17 00:00:00 2001 From: Luck Date: Wed, 13 Jul 2022 21:10:28 +0100 Subject: Support linux x64 musl --- .../common/sampler/async/AsyncProfilerAccess.java | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java index d642a53..ef2c035 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java @@ -29,13 +29,16 @@ import me.lucko.spark.common.util.TemporaryFiles; import one.profiler.AsyncProfiler; import one.profiler.Events; +import java.io.BufferedReader; import java.io.InputStream; +import java.io.InputStreamReader; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Locale; import java.util.logging.Level; +import java.util.stream.Collectors; /** * Provides a bridge between spark and async-profiler. @@ -108,8 +111,13 @@ public enum AsyncProfilerAccess { String os = System.getProperty("os.name").toLowerCase(Locale.ROOT).replace(" ", ""); String arch = System.getProperty("os.arch").toLowerCase(Locale.ROOT); + if (os.equals("linux") && arch.equals("amd64") && isLinuxMusl()) { + arch = "amd64-musl"; + } + Table supported = ImmutableTable.builder() .put("linux", "amd64", "linux/amd64") + .put("linux", "amd64-musl", "linux/amd64-musl") .put("linux", "aarch64", "linux/aarch64") .put("macosx", "amd64", "macos") .put("macosx", "aarch64", "macos") @@ -190,4 +198,20 @@ public enum AsyncProfilerAccess { super("A runtime error occurred whilst loading the native library", cause); } } + + // Checks if the system is using musl instead of glibc + private static boolean isLinuxMusl() { + try { + InputStream stream = new ProcessBuilder("sh", "-c", "ldd `which ls`") + .start() + .getInputStream(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); + String output = reader.lines().collect(Collectors.joining()); + return output.contains("musl"); // shrug + } catch (Throwable e) { + // ignore + return false; + } + } } -- cgit From 319aae27ad290338a5558ac53517e144254a86ce Mon Sep 17 00:00:00 2001 From: Luck Date: Sun, 17 Jul 2022 12:19:03 +0100 Subject: Fix fabric client startup error --- .../java/me/lucko/spark/common/sampler/ThreadDumper.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java index fe3a6a7..fd0c413 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java @@ -76,10 +76,24 @@ public interface ThreadDumper { * the game (server/client) thread. */ final class GameThread implements Supplier { + private Supplier threadSupplier; private Specific dumper = null; + public GameThread() { + + } + + public GameThread(Supplier threadSupplier) { + this.threadSupplier = threadSupplier; + } + @Override public ThreadDumper get() { + if (this.dumper == null) { + setThread(this.threadSupplier.get()); + this.threadSupplier = null; + } + return Objects.requireNonNull(this.dumper, "dumper"); } -- cgit From 618230b958d7822985e2702cd9528f1b4567e59c Mon Sep 17 00:00:00 2001 From: Luck Date: Sun, 4 Sep 2022 10:53:06 +0100 Subject: Improve debug output when JFR parsing fails --- .../spark/common/sampler/async/AsyncSampler.java | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index d8288da..dae3852 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -185,8 +185,19 @@ public class AsyncSampler extends AbstractSampler { // read the jfr file produced by async-profiler try (JfrReader reader = new JfrReader(this.outputFile)) { readSegments(reader, threadFilter); - } catch (IOException e) { - throw new RuntimeException("Read error", e); + } catch (Exception e) { + boolean fileExists; + try { + fileExists = Files.exists(this.outputFile) && Files.size(this.outputFile) != 0; + } catch (IOException ex) { + fileExists = false; + } + + if (fileExists) { + throw new JfrParsingException("Error parsing JFR data from profiler output", e); + } else { + throw new JfrParsingException("Error parsing JFR data from profiler output - file " + this.outputFile + " does not exist!", e); + } } // delete the output file after reading @@ -268,4 +279,10 @@ public class AsyncSampler extends AbstractSampler { reader.stackFrames.put(methodId, result); return result; } + + private static final class JfrParsingException extends RuntimeException { + public JfrParsingException(String message, Throwable cause) { + super(message, cause); + } + } } -- cgit From 7ef9b6281135ce0a24f3c14c2255d9a2c2eca969 Mon Sep 17 00:00:00 2001 From: ishland Date: Mon, 19 Sep 2022 21:48:28 +0800 Subject: Display source info for mixin injected methods (#249) Co-authored-by: Luck --- .../java/me/lucko/spark/common/sampler/AbstractSampler.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index 1c217db..3cfef0b 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -164,8 +164,16 @@ public abstract class AbstractSampler implements Sampler { classSourceVisitor.visit(entry); } - if (classSourceVisitor.hasMappings()) { - proto.putAllClassSources(classSourceVisitor.getMapping()); + if (classSourceVisitor.hasClassSourceMappings()) { + proto.putAllClassSources(classSourceVisitor.getClassSourceMapping()); + } + + if (classSourceVisitor.hasMethodSourceMappings()) { + proto.putAllMethodSources(classSourceVisitor.getMethodSourceMapping()); + } + + if (classSourceVisitor.hasLineSourceMappings()) { + proto.putAllLineSources(classSourceVisitor.getLineSourceMapping()); } } } -- cgit From 7079484d428321c9b3db09394577efda4d591a4e Mon Sep 17 00:00:00 2001 From: Luck Date: Mon, 19 Sep 2022 18:57:02 +0100 Subject: Provide extra metadata about sources in sampler data --- .../spark/common/sampler/AbstractSampler.java | 10 +- .../me/lucko/spark/common/sampler/Sampler.java | 2 +- .../spark/common/sampler/async/AsyncSampler.java | 2 +- .../spark/common/sampler/java/JavaSampler.java | 2 +- .../common/sampler/source/ClassSourceLookup.java | 463 +++++++++++++++++++++ .../common/sampler/source/SourceMetadata.java | 81 ++++ 6 files changed, 556 insertions(+), 4 deletions(-) create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/source/ClassSourceLookup.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/source/SourceMetadata.java (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index 3cfef0b..7b57504 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -27,13 +27,16 @@ import me.lucko.spark.common.platform.serverconfig.ServerConfigProvider; import me.lucko.spark.common.sampler.aggregator.DataAggregator; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; +import me.lucko.spark.common.sampler.source.ClassSourceLookup; +import me.lucko.spark.common.sampler.source.SourceMetadata; import me.lucko.spark.common.tick.TickHook; -import me.lucko.spark.common.util.ClassSourceLookup; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; +import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -150,6 +153,11 @@ public abstract class AbstractSampler implements Sampler { e.printStackTrace(); } + Collection knownSources = platform.getPlugin().getKnownSources(); + for (SourceMetadata source : knownSources) { + metadata.putSources(source.getName().toLowerCase(Locale.ROOT), source.toProto()); + } + proto.setMetadata(metadata); } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java index 84f2da1..98281de 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java @@ -24,7 +24,7 @@ import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.common.util.ClassSourceLookup; +import me.lucko.spark.common.sampler.source.ClassSourceLookup; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import java.util.Comparator; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index dae3852..37ccd96 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -30,7 +30,7 @@ import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.async.jfr.JfrReader; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.common.util.ClassSourceLookup; +import me.lucko.spark.common.sampler.source.ClassSourceLookup; import me.lucko.spark.common.util.TemporaryFiles; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java index 913faee..0f73a9f 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java @@ -29,8 +29,8 @@ import me.lucko.spark.common.sampler.ThreadDumper; import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; +import me.lucko.spark.common.sampler.source.ClassSourceLookup; import me.lucko.spark.common.tick.TickHook; -import me.lucko.spark.common.util.ClassSourceLookup; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import java.lang.management.ManagementFactory; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/source/ClassSourceLookup.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/source/ClassSourceLookup.java new file mode 100644 index 0000000..66b41d2 --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/source/ClassSourceLookup.java @@ -0,0 +1,463 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package me.lucko.spark.common.sampler.source; + +import me.lucko.spark.common.SparkPlatform; +import me.lucko.spark.common.sampler.node.StackTraceNode; +import me.lucko.spark.common.sampler.node.ThreadNode; +import me.lucko.spark.common.util.ClassFinder; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.security.ProtectionDomain; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A function which defines the source of given {@link Class}es or (Mixin) method calls. + */ +public interface ClassSourceLookup { + + /** + * Identify the given class. + * + * @param clazz the class + * @return the source of the class + */ + @Nullable String identify(Class clazz) throws Exception; + + /** + * Identify the given method call. + * + * @param methodCall the method call info + * @return the source of the method call + */ + default @Nullable String identify(MethodCall methodCall) throws Exception { + return null; + } + + /** + * Identify the given method call. + * + * @param methodCall the method call info + * @return the source of the method call + */ + default @Nullable String identify(MethodCallByLine methodCall) throws Exception { + return null; + } + + /** + * A no-operation {@link ClassSourceLookup}. + */ + ClassSourceLookup NO_OP = new ClassSourceLookup() { + @Override + public @Nullable String identify(Class clazz) { + return null; + } + }; + + static ClassSourceLookup create(SparkPlatform platform) { + try { + return platform.createClassSourceLookup(); + } catch (Exception e) { + e.printStackTrace(); + return NO_OP; + } + } + + /** + * A {@link ClassSourceLookup} which identifies classes based on their {@link ClassLoader}. + */ + abstract class ByClassLoader implements ClassSourceLookup { + + public abstract @Nullable String identify(ClassLoader loader) throws Exception; + + @Override + public final @Nullable String identify(Class clazz) throws Exception { + ClassLoader loader = clazz.getClassLoader(); + while (loader != null) { + String source = identify(loader); + if (source != null) { + return source; + } + loader = loader.getParent(); + } + return null; + } + } + + /** + * A {@link ClassSourceLookup} which identifies classes based on URL. + */ + interface ByUrl extends ClassSourceLookup { + + default String identifyUrl(URL url) throws URISyntaxException, MalformedURLException { + Path path = null; + + String protocol = url.getProtocol(); + if (protocol.equals("file")) { + path = Paths.get(url.toURI()); + } else if (protocol.equals("jar")) { + URL innerUrl = new URL(url.getPath()); + path = Paths.get(innerUrl.getPath().split("!")[0]); + } + + if (path != null) { + return identifyFile(path.toAbsolutePath().normalize()); + } + + return null; + } + + default String identifyFile(Path path) { + return identifyFileName(path.getFileName().toString()); + } + + default String identifyFileName(String fileName) { + return fileName.endsWith(".jar") ? fileName.substring(0, fileName.length() - 4) : null; + } + } + + /** + * A {@link ClassSourceLookup} which identifies classes based on the first URL in a {@link URLClassLoader}. + */ + class ByFirstUrlSource extends ClassSourceLookup.ByClassLoader implements ClassSourceLookup.ByUrl { + @Override + public @Nullable String identify(ClassLoader loader) throws IOException, URISyntaxException { + if (loader instanceof URLClassLoader) { + URLClassLoader urlClassLoader = (URLClassLoader) loader; + URL[] urls = urlClassLoader.getURLs(); + if (urls.length == 0) { + return null; + } + return identifyUrl(urls[0]); + } + return null; + } + } + + /** + * A {@link ClassSourceLookup} which identifies classes based on their {@link ProtectionDomain#getCodeSource()}. + */ + class ByCodeSource implements ClassSourceLookup, ClassSourceLookup.ByUrl { + @Override + public @Nullable String identify(Class clazz) throws URISyntaxException, MalformedURLException { + ProtectionDomain protectionDomain = clazz.getProtectionDomain(); + if (protectionDomain == null) { + return null; + } + CodeSource codeSource = protectionDomain.getCodeSource(); + if (codeSource == null) { + return null; + } + + URL url = codeSource.getLocation(); + return url == null ? null : identifyUrl(url); + } + } + + interface Visitor { + void visit(ThreadNode node); + + boolean hasClassSourceMappings(); + + Map getClassSourceMapping(); + + boolean hasMethodSourceMappings(); + + Map getMethodSourceMapping(); + + boolean hasLineSourceMappings(); + + Map getLineSourceMapping(); + } + + static Visitor createVisitor(ClassSourceLookup lookup) { + if (lookup == ClassSourceLookup.NO_OP) { + return NoOpVisitor.INSTANCE; // don't bother! + } + return new VisitorImpl(lookup); + } + + enum NoOpVisitor implements Visitor { + INSTANCE; + + @Override + public void visit(ThreadNode node) { + + } + + @Override + public boolean hasClassSourceMappings() { + return false; + } + + @Override + public Map getClassSourceMapping() { + return Collections.emptyMap(); + } + + @Override + public boolean hasMethodSourceMappings() { + return false; + } + + @Override + public Map getMethodSourceMapping() { + return Collections.emptyMap(); + } + + @Override + public boolean hasLineSourceMappings() { + return false; + } + + @Override + public Map getLineSourceMapping() { + return Collections.emptyMap(); + } + } + + /** + * Visitor which scans {@link StackTraceNode}s and accumulates class/method call identities. + */ + class VisitorImpl implements Visitor { + private final ClassSourceLookup lookup; + private final ClassFinder classFinder = new ClassFinder(); + + private final SourcesMap classSources = new SourcesMap<>(Function.identity()); + private final SourcesMap methodSources = new SourcesMap<>(MethodCall::toString); + private final SourcesMap lineSources = new SourcesMap<>(MethodCallByLine::toString); + + VisitorImpl(ClassSourceLookup lookup) { + this.lookup = lookup; + } + + @Override + public void visit(ThreadNode node) { + for (StackTraceNode child : node.getChildren()) { + visitStackNode(child); + } + } + + private void visitStackNode(StackTraceNode node) { + this.classSources.computeIfAbsent( + node.getClassName(), + className -> { + Class clazz = this.classFinder.findClass(className); + if (clazz == null) { + return null; + } + return this.lookup.identify(clazz); + }); + + if (node.getMethodDescription() != null) { + MethodCall methodCall = new MethodCall(node.getClassName(), node.getMethodName(), node.getMethodDescription()); + this.methodSources.computeIfAbsent(methodCall, this.lookup::identify); + } else { + MethodCallByLine methodCall = new MethodCallByLine(node.getClassName(), node.getMethodName(), node.getLineNumber()); + this.lineSources.computeIfAbsent(methodCall, this.lookup::identify); + } + + // recursively + for (StackTraceNode child : node.getChildren()) { + visitStackNode(child); + } + } + + @Override + public boolean hasClassSourceMappings() { + return this.classSources.hasMappings(); + } + + @Override + public Map getClassSourceMapping() { + return this.classSources.export(); + } + + @Override + public boolean hasMethodSourceMappings() { + return this.methodSources.hasMappings(); + } + + @Override + public Map getMethodSourceMapping() { + return this.methodSources.export(); + } + + @Override + public boolean hasLineSourceMappings() { + return this.lineSources.hasMappings(); + } + + @Override + public Map getLineSourceMapping() { + return this.lineSources.export(); + } + } + + final class SourcesMap { + // --> identifier (plugin name) + private final Map map = new HashMap<>(); + private final Function keyToStringFunction; + + private SourcesMap(Function keyToStringFunction) { + this.keyToStringFunction = keyToStringFunction; + } + + public void computeIfAbsent(T key, ComputeSourceFunction function) { + if (!this.map.containsKey(key)) { + try { + this.map.put(key, function.compute(key)); + } catch (Throwable e) { + this.map.put(key, null); + } + } + } + + public boolean hasMappings() { + this.map.values().removeIf(Objects::isNull); + return !this.map.isEmpty(); + } + + public Map export() { + this.map.values().removeIf(Objects::isNull); + if (this.keyToStringFunction.equals(Function.identity())) { + //noinspection unchecked + return (Map) this.map; + } else { + return this.map.entrySet().stream().collect(Collectors.toMap( + e -> this.keyToStringFunction.apply(e.getKey()), + Map.Entry::getValue + )); + } + } + + private interface ComputeSourceFunction { + String compute(T key) throws Exception; + } + } + + /** + * Encapsulates information about a given method call using the name + method description. + */ + final class MethodCall { + private final String className; + private final String methodName; + private final String methodDescriptor; + + public MethodCall(String className, String methodName, String methodDescriptor) { + this.className = className; + this.methodName = methodName; + this.methodDescriptor = methodDescriptor; + } + + public String getClassName() { + return this.className; + } + + public String getMethodName() { + return this.methodName; + } + + public String getMethodDescriptor() { + return this.methodDescriptor; + } + + @Override + public String toString() { + return this.className + ";" + this.methodName + ";" + this.methodDescriptor; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MethodCall)) return false; + MethodCall that = (MethodCall) o; + return this.className.equals(that.className) && + this.methodName.equals(that.methodName) && + this.methodDescriptor.equals(that.methodDescriptor); + } + + @Override + public int hashCode() { + return Objects.hash(this.className, this.methodName, this.methodDescriptor); + } + } + + /** + * Encapsulates information about a given method call using the name + line number. + */ + final class MethodCallByLine { + private final String className; + private final String methodName; + private final int lineNumber; + + public MethodCallByLine(String className, String methodName, int lineNumber) { + this.className = className; + this.methodName = methodName; + this.lineNumber = lineNumber; + } + + public String getClassName() { + return this.className; + } + + public String getMethodName() { + return this.methodName; + } + + public int getLineNumber() { + return this.lineNumber; + } + + @Override + public String toString() { + return this.className + ";" + this.lineNumber; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MethodCallByLine)) return false; + MethodCallByLine that = (MethodCallByLine) o; + return this.lineNumber == that.lineNumber && this.className.equals(that.className); + } + + @Override + public int hashCode() { + return Objects.hash(this.className, this.lineNumber); + } + } + +} diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/source/SourceMetadata.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/source/SourceMetadata.java new file mode 100644 index 0000000..0808d66 --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/source/SourceMetadata.java @@ -0,0 +1,81 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package me.lucko.spark.common.sampler.source; + +import com.google.common.collect.ImmutableList; + +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +/** + * A "source" is a plugin or mod on the platform that may be identified + * as a source of a method call in a profile. + */ +public class SourceMetadata { + + public static List gather(Collection sources, Function nameFunction, Function versionFunction, Function authorFunction) { + ImmutableList.Builder builder = ImmutableList.builder(); + + for (T source : sources) { + String name = nameFunction.apply(source); + String version = versionFunction.apply(source); + String author = authorFunction.apply(source); + + SourceMetadata metadata = new SourceMetadata(name, version, author); + builder.add(metadata); + } + + return builder.build(); + } + + private final String name; + private final String version; + private final String author; + + public SourceMetadata(String name, String version, String author) { + this.name = name; + this.version = version; + this.author = author; + } + + public String getName() { + return this.name; + } + + public String getVersion() { + return this.version; + } + + public String getAuthor() { + return this.author; + } + + public SamplerMetadata.SourceMetadata toProto() { + return SamplerMetadata.SourceMetadata.newBuilder() + .setName(this.name) + .setVersion(this.version) + .build(); + } + +} -- cgit From a42dda9eebdc8db6c310978d138708c367f95096 Mon Sep 17 00:00:00 2001 From: Luck Date: Mon, 19 Sep 2022 20:08:10 +0100 Subject: Fix issues with temporary files going missing (#225) --- .../lucko/spark/common/sampler/SamplerBuilder.java | 2 +- .../common/sampler/async/AsyncProfilerAccess.java | 30 ++++++++++++++-------- .../spark/common/sampler/async/AsyncSampler.java | 7 +++-- 3 files changed, 24 insertions(+), 15 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java index 88b9d91..52a7387 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/SamplerBuilder.java @@ -98,7 +98,7 @@ public class SamplerBuilder { Sampler sampler; if (this.ticksOver != -1 && this.tickHook != null) { sampler = new JavaSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative, this.tickHook, this.ticksOver); - } else if (this.useAsyncProfiler && !(this.threadDumper instanceof ThreadDumper.Regex) && AsyncProfilerAccess.INSTANCE.checkSupported(platform)) { + } else if (this.useAsyncProfiler && !(this.threadDumper instanceof ThreadDumper.Regex) && AsyncProfilerAccess.getInstance(platform).checkSupported(platform)) { sampler = new AsyncSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout); } else { sampler = new JavaSampler(platform, intervalMicros, this.threadDumper, this.threadGrouper, this.timeout, this.ignoreSleeping, this.ignoreNative); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java index ef2c035..abde21d 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java @@ -22,9 +22,9 @@ package me.lucko.spark.common.sampler.async; import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; +import com.google.common.io.ByteStreams; import me.lucko.spark.common.SparkPlatform; -import me.lucko.spark.common.util.TemporaryFiles; import one.profiler.AsyncProfiler; import one.profiler.Events; @@ -32,19 +32,29 @@ import one.profiler.Events; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.util.Locale; +import java.util.Objects; import java.util.logging.Level; import java.util.stream.Collectors; /** * Provides a bridge between spark and async-profiler. */ -public enum AsyncProfilerAccess { - INSTANCE; +public class AsyncProfilerAccess { + private static AsyncProfilerAccess instance; + + // singleton, needs a SparkPlatform for first init + public static synchronized AsyncProfilerAccess getInstance(SparkPlatform platform) { + if (instance == null) { + Objects.requireNonNull(platform, "platform"); + instance = new AsyncProfilerAccess(platform); + } + return instance; + } /** An instance of the async-profiler Java API. */ private final AsyncProfiler profiler; @@ -55,13 +65,13 @@ public enum AsyncProfilerAccess { /** If profiler is null, contains the reason why setup failed */ private final Exception setupException; - AsyncProfilerAccess() { + AsyncProfilerAccess(SparkPlatform platform) { AsyncProfiler profiler; ProfilingEvent profilingEvent = null; Exception setupException = null; try { - profiler = load(); + profiler = load(platform); if (isEventSupported(profiler, ProfilingEvent.CPU, false)) { profilingEvent = ProfilingEvent.CPU; } else if (isEventSupported(profiler, ProfilingEvent.WALL, true)) { @@ -106,7 +116,7 @@ public enum AsyncProfilerAccess { return this.profiler != null; } - private static AsyncProfiler load() throws Exception { + private static AsyncProfiler load(SparkPlatform platform) throws Exception { // check compatibility String os = System.getProperty("os.name").toLowerCase(Locale.ROOT).replace(" ", ""); String arch = System.getProperty("os.arch").toLowerCase(Locale.ROOT); @@ -135,10 +145,10 @@ public enum AsyncProfilerAccess { throw new IllegalStateException("Could not find " + resource + " in spark jar file"); } - Path extractPath = TemporaryFiles.create("spark-", "-libasyncProfiler.so.tmp"); + Path extractPath = platform.getTemporaryFiles().create("spark-", "-libasyncProfiler.so.tmp"); - try (InputStream in = profilerResource.openStream()) { - Files.copy(in, extractPath, StandardCopyOption.REPLACE_EXISTING); + try (InputStream in = profilerResource.openStream(); OutputStream out = Files.newOutputStream(extractPath)) { + ByteStreams.copy(in, out); } // get an instance of async-profiler diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index 37ccd96..7d9cb81 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -31,7 +31,6 @@ import me.lucko.spark.common.sampler.async.jfr.JfrReader; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.sampler.source.ClassSourceLookup; -import me.lucko.spark.common.util.TemporaryFiles; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import one.profiler.AsyncProfiler; @@ -67,7 +66,7 @@ public class AsyncSampler extends AbstractSampler { public AsyncSampler(SparkPlatform platform, int interval, ThreadDumper threadDumper, ThreadGrouper threadGrouper, long endTime) { super(platform, interval, threadDumper, endTime); - this.profiler = AsyncProfilerAccess.INSTANCE.getProfiler(); + this.profiler = AsyncProfilerAccess.getInstance(platform).getProfiler(); this.dataAggregator = new AsyncDataAggregator(threadGrouper); } @@ -93,12 +92,12 @@ public class AsyncSampler extends AbstractSampler { super.start(); try { - this.outputFile = TemporaryFiles.create("spark-profile-", ".jfr.tmp"); + this.outputFile = this.platform.getTemporaryFiles().create("spark-", "-profile-data.jfr.tmp"); } catch (IOException e) { throw new RuntimeException("Unable to create temporary output file", e); } - String command = "start,event=" + AsyncProfilerAccess.INSTANCE.getProfilingEvent() + ",interval=" + this.interval + "us,threads,jfr,file=" + this.outputFile.toString(); + String command = "start,event=" + AsyncProfilerAccess.getInstance(this.platform).getProfilingEvent() + ",interval=" + this.interval + "us,threads,jfr,file=" + this.outputFile.toString(); if (this.threadDumper instanceof ThreadDumper.Specific) { command += ",filter"; } -- cgit From dbdd3eb1344b837abb13538b9c55d1d99e697e54 Mon Sep 17 00:00:00 2001 From: Luck Date: Thu, 22 Sep 2022 22:06:10 +0100 Subject: Allow platforms to pass extra misc metadata to the viewer --- .../java/me/lucko/spark/common/sampler/AbstractSampler.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index 7b57504..e20a2a8 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -23,6 +23,7 @@ package me.lucko.spark.common.sampler; import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics; +import me.lucko.spark.common.platform.MetadataProvider; import me.lucko.spark.common.platform.serverconfig.ServerConfigProvider; import me.lucko.spark.common.sampler.aggregator.DataAggregator; import me.lucko.spark.common.sampler.node.MergeMode; @@ -148,7 +149,14 @@ public abstract class AbstractSampler implements Sampler { try { ServerConfigProvider serverConfigProvider = platform.getPlugin().createServerConfigProvider(); - metadata.putAllServerConfigurations(serverConfigProvider.exportServerConfigurations()); + metadata.putAllServerConfigurations(serverConfigProvider.export()); + } catch (Exception e) { + e.printStackTrace(); + } + + try { + MetadataProvider extraMetadataProvider = platform.getPlugin().createExtraMetadataProvider(); + metadata.putAllExtraPlatformMetadata(extraMetadataProvider.export()); } catch (Exception e) { e.printStackTrace(); } -- cgit From c4b1eccd9cd51e348983fab42ced78166f39cb0e Mon Sep 17 00:00:00 2001 From: Luck Date: Sun, 2 Oct 2022 20:16:03 +0100 Subject: Fix NPE caused by extraMetadataProvider being null --- .../src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index e20a2a8..6fc5a10 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -156,7 +156,9 @@ public abstract class AbstractSampler implements Sampler { try { MetadataProvider extraMetadataProvider = platform.getPlugin().createExtraMetadataProvider(); - metadata.putAllExtraPlatformMetadata(extraMetadataProvider.export()); + if (extraMetadataProvider != null) { + metadata.putAllExtraPlatformMetadata(extraMetadataProvider.export()); + } } catch (Exception e) { e.printStackTrace(); } -- cgit From d31f3c7bdf03c874ff9518d47d060adc18322d6b Mon Sep 17 00:00:00 2001 From: lucko Date: Fri, 7 Oct 2022 20:26:24 +0100 Subject: Split profiler output into windows (#253) --- .../spark/common/sampler/AbstractSampler.java | 42 ++-- .../me/lucko/spark/common/sampler/Sampler.java | 4 +- .../spark/common/sampler/ThreadNodeOrder.java | 52 ---- .../common/sampler/async/AsyncDataAggregator.java | 4 +- .../common/sampler/async/AsyncProfilerAccess.java | 4 +- .../common/sampler/async/AsyncProfilerJob.java | 264 ++++++++++++++++++++ .../spark/common/sampler/async/AsyncSampler.java | 255 ++++++-------------- .../common/sampler/async/JfrParsingException.java | 27 +++ .../spark/common/sampler/async/ProfileSegment.java | 50 ++++ .../spark/common/sampler/async/jfr/Dictionary.java | 4 + .../common/sampler/java/JavaDataAggregator.java | 7 +- .../spark/common/sampler/java/JavaSampler.java | 56 ++++- .../common/sampler/java/SimpleDataAggregator.java | 4 +- .../common/sampler/java/TickedDataAggregator.java | 41 ++-- .../spark/common/sampler/node/AbstractNode.java | 70 +++--- .../spark/common/sampler/node/StackTraceNode.java | 77 +----- .../spark/common/sampler/node/ThreadNode.java | 42 +++- .../sampler/window/ProfilingWindowUtils.java | 36 +++ .../common/sampler/window/ProtoTimeEncoder.java | 94 ++++++++ .../sampler/window/WindowStatisticsCollector.java | 267 +++++++++++++++++++++ 20 files changed, 1000 insertions(+), 400 deletions(-) delete mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadNodeOrder.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerJob.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/async/JfrParsingException.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/window/ProfilingWindowUtils.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/window/ProtoTimeEncoder.java create mode 100644 spark-common/src/main/java/me/lucko/spark/common/sampler/window/WindowStatisticsCollector.java (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index 6fc5a10..c650738 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -30,7 +30,10 @@ import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.sampler.source.ClassSourceLookup; import me.lucko.spark.common.sampler.source.SourceMetadata; +import me.lucko.spark.common.sampler.window.ProtoTimeEncoder; +import me.lucko.spark.common.sampler.window.WindowStatisticsCollector; import me.lucko.spark.common.tick.TickHook; +import me.lucko.spark.proto.SparkProtos; import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; @@ -58,12 +61,12 @@ public abstract class AbstractSampler implements Sampler { /** The time when sampling first began */ protected long startTime = -1; - /** The game tick when sampling first began */ - protected int startTick = -1; - /** The unix timestamp (in millis) when this sampler should automatically complete. */ protected final long autoEndTime; // -1 for nothing + /** Collects statistics for each window in the sample */ + protected final WindowStatisticsCollector windowStatisticsCollector; + /** A future to encapsulate the completion of this sampler instance */ protected final CompletableFuture future = new CompletableFuture<>(); @@ -75,6 +78,7 @@ public abstract class AbstractSampler implements Sampler { this.interval = interval; this.threadDumper = threadDumper; this.autoEndTime = autoEndTime; + this.windowStatisticsCollector = new WindowStatisticsCollector(platform); } @Override @@ -106,11 +110,11 @@ public abstract class AbstractSampler implements Sampler { @Override public void start() { this.startTime = System.currentTimeMillis(); + } - TickHook tickHook = this.platform.getTickHook(); - if (tickHook != null) { - this.startTick = tickHook.getCurrentTick(); - } + @Override + public void stop() { + this.windowStatisticsCollector.stop(); } protected void writeMetadataToProto(SamplerData.Builder proto, SparkPlatform platform, CommandSender creator, String comment, DataAggregator dataAggregator) { @@ -127,12 +131,9 @@ public abstract class AbstractSampler implements Sampler { metadata.setComment(comment); } - if (this.startTick != -1) { - TickHook tickHook = this.platform.getTickHook(); - if (tickHook != null) { - int numberOfTicks = tickHook.getCurrentTick() - this.startTick; - metadata.setNumberOfTicks(numberOfTicks); - } + int totalTicks = this.windowStatisticsCollector.getTotalTicks(); + if (totalTicks != -1) { + metadata.setNumberOfTicks(totalTicks); } try { @@ -171,14 +172,23 @@ public abstract class AbstractSampler implements Sampler { proto.setMetadata(metadata); } - protected void writeDataToProto(SamplerData.Builder proto, DataAggregator dataAggregator, Comparator outputOrder, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { + protected void writeDataToProto(SamplerData.Builder proto, DataAggregator dataAggregator, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { List data = dataAggregator.exportData(); - data.sort(outputOrder); + data.sort(Comparator.comparing(ThreadNode::getThreadLabel)); ClassSourceLookup.Visitor classSourceVisitor = ClassSourceLookup.createVisitor(classSourceLookup); + ProtoTimeEncoder timeEncoder = new ProtoTimeEncoder(data); + int[] timeWindows = timeEncoder.getKeys(); + for (int timeWindow : timeWindows) { + proto.addTimeWindows(timeWindow); + } + + this.windowStatisticsCollector.ensureHasStatisticsForAllWindows(timeWindows); + proto.putAllTimeWindowStatistics(this.windowStatisticsCollector.export()); + for (ThreadNode entry : data) { - proto.addThreads(entry.toProto(mergeMode)); + proto.addThreads(entry.toProto(mergeMode, timeEncoder)); classSourceVisitor.visit(entry); } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java index 98281de..e06cba6 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java @@ -23,11 +23,9 @@ package me.lucko.spark.common.sampler; import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; import me.lucko.spark.common.sampler.node.MergeMode; -import me.lucko