From bfbbcb3e68e019da4657ef0da22b889de656ae3f Mon Sep 17 00:00:00 2001 From: Luck Date: Tue, 28 Dec 2021 18:12:33 +0000 Subject: Include platform and system statistics in profiler viewer payload --- .../me/lucko/spark/common/sampler/AbstractSampler.java | 14 ++++++++++++++ .../main/java/me/lucko/spark/common/sampler/Sampler.java | 6 +++--- .../java/me/lucko/spark/common/sampler/ThreadDumper.java | 2 +- .../me/lucko/spark/common/sampler/ThreadGrouper.java | 2 +- .../spark/common/sampler/aggregator/DataAggregator.java | 2 +- .../spark/common/sampler/async/AsyncDataAggregator.java | 2 +- .../spark/common/sampler/async/AsyncProfilerAccess.java | 2 +- .../lucko/spark/common/sampler/async/AsyncSampler.java | 16 ++++++++++------ .../me/lucko/spark/common/sampler/java/JavaSampler.java | 12 +++++++----- .../spark/common/sampler/java/SimpleDataAggregator.java | 2 +- .../spark/common/sampler/java/TickedDataAggregator.java | 2 +- .../lucko/spark/common/sampler/node/StackTraceNode.java | 6 +++--- .../me/lucko/spark/common/sampler/node/ThreadNode.java | 6 +++--- 13 files changed, 47 insertions(+), 27 deletions(-) (limited to 'spark-common/src/main/java/me/lucko/spark/common/sampler') diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index bae93b1..568b59d 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -20,6 +20,9 @@ package me.lucko.spark.common.sampler; +import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics; + +import java.util.Map; import java.util.concurrent.CompletableFuture; /** @@ -42,6 +45,9 @@ public abstract class AbstractSampler implements Sampler { /** A future to encapsulate the completion of this sampler instance */ protected final CompletableFuture future = new CompletableFuture<>(); + /** The garbage collector statistics when profiling started */ + protected Map initialGcStats; + protected AbstractSampler(int interval, ThreadDumper threadDumper, long endTime) { this.interval = interval; this.threadDumper = threadDumper; @@ -65,4 +71,12 @@ public abstract class AbstractSampler implements Sampler { public CompletableFuture getFuture() { return this.future; } + + protected void recordInitialGcStats() { + this.initialGcStats = GarbageCollectorStatistics.pollStats(); + } + + protected Map getInitialGcStats() { + return this.initialGcStats; + } } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java index b71aaee..d27b2fc 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java @@ -20,12 +20,12 @@ package me.lucko.spark.common.sampler; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.util.ClassSourceLookup; -import me.lucko.spark.proto.SparkProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import java.util.Comparator; import java.util.Map; @@ -68,6 +68,6 @@ public interface Sampler { CompletableFuture getFuture(); // Methods used to export the sampler data to the web viewer. - SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup); + SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup); } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java index e99114a..5cc41b9 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler; import me.lucko.spark.common.util.ThreadFinder; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java index e63ebc8..225f768 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java @@ -20,7 +20,7 @@ package me.lucko.spark.common.sampler; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java index 8b90639..3b1d349 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java @@ -21,7 +21,7 @@ package me.lucko.spark.common.sampler.aggregator; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.util.Map; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java index 594d56e..3de3943 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java @@ -24,7 +24,7 @@ import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.AbstractDataAggregator; import me.lucko.spark.common.sampler.node.StackTraceNode; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; /** * Data aggregator for {@link AsyncSampler}. diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java index f1d7209..06db795 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java @@ -102,7 +102,7 @@ public enum AsyncProfilerAccess { } // extract the profiler binary from the spark jar file - String resource = os + "/libasyncProfiler.so"; + String resource = "spark/" + os + "/libasyncProfiler.so"; URL profilerResource = AsyncProfilerAccess.class.getClassLoader().getResource(resource); if (profilerResource == null) { throw new IllegalStateException("Could not find " + resource + " in spark jar file"); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index 1837cbc..db8802c 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -22,8 +22,8 @@ package me.lucko.spark.common.sampler.async; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.AbstractSampler; import me.lucko.spark.common.sampler.ThreadDumper; import me.lucko.spark.common.sampler.ThreadGrouper; @@ -32,7 +32,8 @@ import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.util.ClassSourceLookup; import me.lucko.spark.common.util.TemporaryFiles; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import one.profiler.AsyncProfiler; @@ -117,6 +118,7 @@ public class AsyncSampler extends AbstractSampler { } } + recordInitialGcStats(); scheduleTimeout(); } @@ -154,9 +156,11 @@ public class AsyncSampler extends AbstractSampler { } @Override - public SparkProtos.SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { - final SparkProtos.SamplerMetadata.Builder metadata = SparkProtos.SamplerMetadata.newBuilder() - .setPlatformMetadata(platformInfo.toData().toProto()) + public SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { + final SamplerMetadata.Builder metadata = SamplerMetadata.newBuilder() + .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) + .setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(getInitialGcStats())) + .setSystemStatistics(platform.getStatisticsProvider().getSystemStatistics()) .setCreator(creator.toData().toProto()) .setStartTime(this.startTime) .setInterval(this.interval) @@ -167,7 +171,7 @@ public class AsyncSampler extends AbstractSampler { metadata.setComment(comment); } - SparkProtos.SamplerData.Builder proto = SparkProtos.SamplerData.newBuilder(); + SamplerData.Builder proto = SamplerData.newBuilder(); proto.setMetadata(metadata.build()); aggregateOutput(); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java index 02d5f01..c873f9f 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java @@ -23,8 +23,8 @@ package me.lucko.spark.common.sampler.java; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.AbstractSampler; import me.lucko.spark.common.sampler.ThreadDumper; import me.lucko.spark.common.sampler.ThreadGrouper; @@ -32,8 +32,8 @@ import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.tick.TickHook; import me.lucko.spark.common.util.ClassSourceLookup; -import me.lucko.spark.proto.SparkProtos.SamplerData; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -129,9 +129,11 @@ public class JavaSampler extends AbstractSampler implements Runnable { } @Override - public SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { + public SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { final SamplerMetadata.Builder metadata = SamplerMetadata.newBuilder() - .setPlatformMetadata(platformInfo.toData().toProto()) + .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) + .setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(getInitialGcStats())) + .setSystemStatistics(platform.getStatisticsProvider().getSystemStatistics()) .setCreator(creator.toData().toProto()) .setStartTime(this.startTime) .setInterval(this.interval) diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java index e7113a1..39e21aa 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler.java; import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.DataAggregator; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.util.concurrent.ExecutorService; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java index 018a3b8..ac34d01 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java @@ -24,7 +24,7 @@ import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.DataAggregator; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.tick.TickHook; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.util.ArrayList; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java index f935fb2..54217be 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler.node; import me.lucko.spark.common.util.MethodDisambiguator; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos; import org.checkerframework.checker.nullness.qual.Nullable; @@ -65,8 +65,8 @@ public final class StackTraceNode extends AbstractNode implements Comparable