diff options
Diffstat (limited to 'spark-common/src/main/java/me/lucko')
27 files changed, 514 insertions, 190 deletions
diff --git a/spark-common/src/main/java/me/lucko/spark/common/SparkPlatform.java b/spark-common/src/main/java/me/lucko/spark/common/SparkPlatform.java index 57f8732..a721adc 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/SparkPlatform.java +++ b/spark-common/src/main/java/me/lucko/spark/common/SparkPlatform.java @@ -41,6 +41,7 @@ import me.lucko.spark.common.command.tabcomplete.TabCompleter; import me.lucko.spark.common.monitor.cpu.CpuMonitor; import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics; import me.lucko.spark.common.monitor.tick.TickStatistics; +import me.lucko.spark.common.platform.PlatformStatisticsProvider; import me.lucko.spark.common.tick.TickHook; import me.lucko.spark.common.tick.TickReporter; import me.lucko.spark.common.util.BytebinClient; @@ -96,6 +97,7 @@ public class SparkPlatform { private final TickHook tickHook; private final TickReporter tickReporter; private final TickStatistics tickStatistics; + private final PlatformStatisticsProvider statisticsProvider; private Map<String, GarbageCollectorStatistics> startupGcStatistics = ImmutableMap.of(); private long serverNormalOperationStartTime; private final AtomicBoolean enabled = new AtomicBoolean(false); @@ -132,6 +134,7 @@ public class SparkPlatform { this.tickHook = plugin.createTickHook(); this.tickReporter = plugin.createTickReporter(); this.tickStatistics = this.tickHook != null ? new TickStatistics() : null; + this.statisticsProvider = new PlatformStatisticsProvider(this); } public void enable() { @@ -214,6 +217,10 @@ public class SparkPlatform { return this.tickReporter; } + public PlatformStatisticsProvider getStatisticsProvider() { + return this.statisticsProvider; + } + public ClassSourceLookup createClassSourceLookup() { return this.plugin.createClassSourceLookup(); } diff --git a/spark-common/src/main/java/me/lucko/spark/common/api/GarbageCollectorInfo.java b/spark-common/src/main/java/me/lucko/spark/common/api/GarbageCollectorInfo.java index 8d289aa..fc14c67 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/api/GarbageCollectorInfo.java +++ b/spark-common/src/main/java/me/lucko/spark/common/api/GarbageCollectorInfo.java @@ -36,10 +36,8 @@ public class GarbageCollectorInfo implements GarbageCollector { this.name = name; this.totalCollections = stats.getCollectionCount(); this.totalTime = stats.getCollectionTime(); - - double totalTimeDouble = this.totalTime; - this.averageTime = this.totalCollections == 0 ? 0 : totalTimeDouble / this.totalCollections; - this.averageFrequency = this.totalCollections == 0 ? 0 : (long) ((serverUptime - totalTimeDouble) / this.totalCollections); + this.averageTime = stats.getAverageCollectionTime(); + this.averageFrequency = stats.getAverageCollectionFrequency(serverUptime); } @Override diff --git a/spark-common/src/main/java/me/lucko/spark/common/command/modules/HealthModule.java b/spark-common/src/main/java/me/lucko/spark/common/command/modules/HealthModule.java index 51fa905..b036d21 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/command/modules/HealthModule.java +++ b/spark-common/src/main/java/me/lucko/spark/common/command/modules/HealthModule.java @@ -30,6 +30,7 @@ import me.lucko.spark.common.command.CommandResponseHandler; import me.lucko.spark.common.command.sender.CommandSender; import me.lucko.spark.common.command.tabcomplete.TabCompleter; import me.lucko.spark.common.monitor.cpu.CpuMonitor; +import me.lucko.spark.common.monitor.disk.DiskUsage; import me.lucko.spark.common.monitor.tick.TickStatistics; import me.lucko.spark.common.util.FormatUtil; import me.lucko.spark.common.util.RollingAverage; @@ -38,15 +39,11 @@ import net.kyori.adventure.text.Component; import net.kyori.adventure.text.TextComponent; import net.kyori.adventure.text.format.TextColor; -import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryType; import java.lang.management.MemoryUsage; -import java.nio.file.FileStore; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.LinkedList; import java.util.List; import java.util.function.Consumer; @@ -65,8 +62,6 @@ import static net.kyori.adventure.text.format.TextDecoration.BOLD; public class HealthModule implements CommandModule { - private static final double MSPT_95_PERCENTILE = 0.95d; - @Override public void registerCommands(Consumer<Command> consumer) { consumer.accept(Command.builder() @@ -150,11 +145,7 @@ public class HealthModule implements CommandModule { addDetailedMemoryStats(report, memoryMXBean); } - try { - addDiskStats(report); - } catch (IOException e) { - e.printStackTrace(); - } + addDiskStats(report); resp.reply(report); } @@ -309,10 +300,14 @@ public class HealthModule implements CommandModule { } } - private static void addDiskStats(List<Component> report) throws IOException { - FileStore fileStore = Files.getFileStore(Paths.get(".")); - long totalSpace = fileStore.getTotalSpace(); - long usedSpace = totalSpace - fileStore.getUsableSpace(); + private static void addDiskStats(List<Component> report) { + long total = DiskUsage.getTotal(); + long used = DiskUsage.getUsed(); + + if (total == 0 || used == 0) { + return; + } + report.add(text() .append(text(">", DARK_GRAY, BOLD)) .append(space()) @@ -321,18 +316,18 @@ public class HealthModule implements CommandModule { ); report.add(text() .content(" ") - .append(text(FormatUtil.formatBytes(usedSpace), WHITE)) + .append(text(FormatUtil.formatBytes(used), WHITE)) .append(space()) .append(text("/", GRAY)) .append(space()) - .append(text(FormatUtil.formatBytes(totalSpace), WHITE)) + .append(text(FormatUtil.formatBytes(total), WHITE)) .append(text(" ")) .append(text("(", GRAY)) - .append(text(FormatUtil.percent(usedSpace, totalSpace), GREEN)) + .append(text(FormatUtil.percent(used, total), GREEN)) .append(text(")", GRAY)) .build() ); - report.add(text().content(" ").append(generateDiskUsageDiagram(usedSpace, totalSpace, 40)).build()); + report.add(text().content(" ").append(generateDiskUsageDiagram(used, total, 40)).build()); report.add(empty()); } @@ -355,7 +350,7 @@ public class HealthModule implements CommandModule { .append(text('/', GRAY)) .append(formatTickDuration(average.median())) .append(text('/', GRAY)) - .append(formatTickDuration(average.percentile(MSPT_95_PERCENTILE))) + .append(formatTickDuration(average.percentile95th())) .append(text('/', GRAY)) .append(formatTickDuration(average.max())) .build(); diff --git a/spark-common/src/main/java/me/lucko/spark/common/command/modules/HeapAnalysisModule.java b/spark-common/src/main/java/me/lucko/spark/common/command/modules/HeapAnalysisModule.java index 70f6c3c..491ec1e 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/command/modules/HeapAnalysisModule.java +++ b/spark-common/src/main/java/me/lucko/spark/common/command/modules/HeapAnalysisModule.java @@ -30,20 +30,15 @@ import me.lucko.spark.common.command.sender.CommandSender; import me.lucko.spark.common.command.tabcomplete.TabCompleter; import me.lucko.spark.common.heapdump.HeapDump; import me.lucko.spark.common.heapdump.HeapDumpSummary; +import me.lucko.spark.common.util.Compression; import me.lucko.spark.common.util.FormatUtil; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkHeapProtos; import net.kyori.adventure.text.event.ClickEvent; -import org.tukaani.xz.LZMA2Options; -import org.tukaani.xz.LZMAOutputStream; -import org.tukaani.xz.XZOutputStream; - import okhttp3.MediaType; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Iterator; @@ -51,7 +46,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.function.LongConsumer; -import java.util.zip.GZIPOutputStream; import static net.kyori.adventure.text.Component.text; import static net.kyori.adventure.text.format.NamedTextColor.GOLD; @@ -98,7 +92,7 @@ public class HeapAnalysisModule implements CommandModule { return; } - SparkProtos.HeapData output = heapDump.toProto(platform.getPlugin().getPlatformInfo(), sender); + SparkHeapProtos.HeapData output = heapDump.toProto(platform, sender); boolean saveToFile = false; if (arguments.boolFlag("save-to-file")) { @@ -175,11 +169,11 @@ public class HeapAnalysisModule implements CommandModule { platform.getActivityLog().addToLog(Activity.fileActivity(sender, System.currentTimeMillis(), "Heap dump", file.toString())); - CompressionMethod compressionMethod = null; + Compression compressionMethod = null; Iterator<String> compressArgs = arguments.stringFlag("compress").iterator(); if (compressArgs.hasNext()) { try { - compressionMethod = CompressionMethod.valueOf(compressArgs.next().toUpperCase()); + compressionMethod = Compression.valueOf(compressArgs.next().toUpperCase()); } catch (IllegalArgumentException e) { // ignore } @@ -194,7 +188,7 @@ public class HeapAnalysisModule implements CommandModule { } } - private static void heapDumpCompress(SparkPlatform platform, CommandResponseHandler resp, Path file, CompressionMethod method) throws IOException { + private static void heapDumpCompress(SparkPlatform platform, CommandResponseHandler resp, Path file, Compression method) throws IOException { resp.broadcastPrefixed(text("Compressing heap dump, please wait...")); long size = Files.size(file); @@ -244,71 +238,4 @@ public class HeapAnalysisModule implements CommandModule { ); } - public enum CompressionMethod { - GZIP { - @Override - public Path compress(Path file, LongConsumer progressHandler) throws IOException { - Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".gz"); - try (InputStream in = Files.newInputStream(file)) { - try (OutputStream out = Files.newOutputStream(compressedFile)) { - try (GZIPOutputStream compressionOut = new GZIPOutputStream(out, 1024 * 64)) { - copy(in, compressionOut, progressHandler); - } - } - } - return compressedFile; - } - }, - XZ { - @Override - public Path compress(Path file, LongConsumer progressHandler) throws IOException { - Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".xz"); - try (InputStream in = Files.newInputStream(file)) { - try (OutputStream out = Files.newOutputStream(compressedFile)) { - try (XZOutputStream compressionOut = new XZOutputStream(out, new LZMA2Options())) { - copy(in, compressionOut, progressHandler); - } - } - } - return compressedFile; - } - }, - LZMA { - @Override - public Path compress(Path file, LongConsumer progressHandler) throws IOException { - Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".lzma"); - try (InputStream in = Files.newInputStream(file)) { - try (OutputStream out = Files.newOutputStream(compressedFile)) { - try (LZMAOutputStream compressionOut = new LZMAOutputStream(out, new LZMA2Options(), true)) { - copy(in, compressionOut, progressHandler); - } - } - } - return compressedFile; - } - }; - - public abstract Path compress(Path file, LongConsumer progressHandler) throws IOException; - - private static long copy(InputStream from, OutputStream to, LongConsumer progress) throws IOException { - byte[] buf = new byte[1024 * 64]; - long total = 0; - long iterations = 0; - while (true) { - int r = from.read(buf); - if (r == -1) { - break; - } - to.write(buf, 0, r); - total += r; - - // report progress every 5MB - if (iterations++ % ((1024 / 64) * 5) == 0) { - progress.accept(total); - } - } - return total; - } - } - } diff --git a/spark-common/src/main/java/me/lucko/spark/common/command/modules/SamplerModule.java b/spark-common/src/main/java/me/lucko/spark/common/command/modules/SamplerModule.java index 2dd07c9..26f20e7 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/command/modules/SamplerModule.java +++ b/spark-common/src/main/java/me/lucko/spark/common/command/modules/SamplerModule.java @@ -40,7 +40,7 @@ import me.lucko.spark.common.sampler.async.AsyncSampler; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.tick.TickHook; import me.lucko.spark.common.util.MethodDisambiguator; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos; import net.kyori.adventure.text.event.ClickEvent; @@ -305,7 +305,7 @@ public class SamplerModule implements CommandModule { } private void handleUpload(SparkPlatform platform, CommandResponseHandler resp, Sampler sampler, ThreadNodeOrder threadOrder, String comment, MergeMode mergeMode, boolean saveToFileFlag) { - SparkProtos.SamplerData output = sampler.toProto(platform.getPlugin().getPlatformInfo(), resp.sender(), threadOrder, comment, mergeMode, platform.createClassSourceLookup()); + SparkSamplerProtos.SamplerData output = sampler.toProto(platform, resp.sender(), threadOrder, comment, mergeMode, platform.createClassSourceLookup()); boolean saveToFile = false; if (saveToFileFlag) { diff --git a/spark-common/src/main/java/me/lucko/spark/common/heapdump/HeapDumpSummary.java b/spark-common/src/main/java/me/lucko/spark/common/heapdump/HeapDumpSummary.java index 34fd6c4..7bb411d 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/heapdump/HeapDumpSummary.java +++ b/spark-common/src/main/java/me/lucko/spark/common/heapdump/HeapDumpSummary.java @@ -20,11 +20,11 @@ package me.lucko.spark.common.heapdump; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; -import me.lucko.spark.proto.SparkProtos; -import me.lucko.spark.proto.SparkProtos.HeapData; -import me.lucko.spark.proto.SparkProtos.HeapEntry; +import me.lucko.spark.proto.SparkHeapProtos.HeapData; +import me.lucko.spark.proto.SparkHeapProtos.HeapEntry; +import me.lucko.spark.proto.SparkHeapProtos.HeapMetadata; import org.objectweb.asm.Type; @@ -125,10 +125,12 @@ public final class HeapDumpSummary { this.entries = entries; } - public HeapData toProto(PlatformInfo platformInfo, CommandSender creator) { + public HeapData toProto(SparkPlatform platform, CommandSender creator) { HeapData.Builder proto = HeapData.newBuilder(); - proto.setMetadata(SparkProtos.HeapMetadata.newBuilder() - .setPlatformMetadata(platformInfo.toData().toProto()) + proto.setMetadata(HeapMetadata.newBuilder() + .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) + .setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(null)) + .setSystemStatistics(platform.getStatisticsProvider().getSystemStatistics()) .setCreator(creator.toData().toProto()) .build() ); diff --git a/spark-common/src/main/java/me/lucko/spark/common/monitor/disk/DiskUsage.java b/spark-common/src/main/java/me/lucko/spark/common/monitor/disk/DiskUsage.java new file mode 100644 index 0000000..4450fcd --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/monitor/disk/DiskUsage.java @@ -0,0 +1,63 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) <luck@lucko.me> + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +package me.lucko.spark.common.monitor.disk; + +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Paths; + +/** + * Exposes the system disk usage. + */ +public enum DiskUsage { + ; + + private static final FileStore FILE_STORE; + + static { + FileStore fileStore = null; + try { + fileStore = Files.getFileStore(Paths.get(".")); + } catch (IOException e) { + // ignore + } + FILE_STORE = fileStore; + } + + public static long getUsed() { + try { + long total = FILE_STORE.getTotalSpace(); + return total - FILE_STORE.getUsableSpace(); + } catch (IOException e) { + return 0; + } + } + + public static long getTotal() { + try { + return FILE_STORE.getTotalSpace(); + } catch (IOException e) { + return 0; + } + } + +} diff --git a/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/GarbageCollectorStatistics.java b/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/GarbageCollectorStatistics.java index c831ea1..cfd12a1 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/GarbageCollectorStatistics.java +++ b/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/GarbageCollectorStatistics.java @@ -74,6 +74,8 @@ public class GarbageCollectorStatistics { this(bean.getCollectionCount(), bean.getCollectionTime()); } + // all times in milliseconds + public long getCollectionCount() { return this.collectionCount; } @@ -82,6 +84,14 @@ public class GarbageCollectorStatistics { return this.collectionTime; } + public double getAverageCollectionTime() { + return this.collectionCount == 0 ? 0 : (double) this.collectionTime / this.collectionCount; + } + + public long getAverageCollectionFrequency(long serverUptime) { + return this.collectionCount == 0 ? 0 : (long) ((serverUptime - (double) this.collectionTime) / this.collectionCount); + } + public GarbageCollectorStatistics subtract(GarbageCollectorStatistics other) { if (other == ZERO || (other.collectionCount == 0 && other.collectionTime == 0)) { return this; diff --git a/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/MemoryInfo.java b/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/MemoryInfo.java new file mode 100644 index 0000000..4ed9b1c --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/monitor/memory/MemoryInfo.java @@ -0,0 +1,74 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) <luck@lucko.me> + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +package me.lucko.spark.common.monitor.memory; + +import java.lang.management.ManagementFactory; + +import javax.management.JMX; +import javax.management.MBeanServer; +import javax.management.ObjectName; + +public enum MemoryInfo { + ; + + /** The object name of the com.sun.management.OperatingSystemMXBean */ + private static final String OPERATING_SYSTEM_BEAN = "java.lang:type=OperatingSystem"; + /** The OperatingSystemMXBean instance */ + private static final OperatingSystemMXBean BEAN; + + static { + try { + MBeanServer beanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName diagnosticBeanName = ObjectName.getInstance(OPERATING_SYSTEM_BEAN); + BEAN = JMX.newMXBeanProxy(beanServer, diagnosticBeanName, OperatingSystemMXBean.class); + } catch (Exception e) { + throw new UnsupportedOperationException("OperatingSystemMXBean is not supported by the system", e); + } + } + + public static long getUsedSwap() { + return BEAN.getTotalSwapSpaceSize() - BEAN.getFreeSwapSpaceSize(); + } + + public static long getTotalSwap() { + return BEAN.getTotalSwapSpaceSize(); + } + + public static long getUsedPhysicalMemory() { + return BEAN.getTotalPhysicalMemorySize() - BEAN.getFreePhysicalMemorySize(); + } + + public static long getTotalPhysicalMemory() { + return BEAN.getTotalPhysicalMemorySize(); + } + + public static long getTotalVirtualMemory() { + return BEAN.getCommittedVirtualMemorySize(); + } + + public interface OperatingSystemMXBean { + long getCommittedVirtualMemorySize(); + long getTotalSwapSpaceSize(); + long getFreeSwapSpaceSize(); + long getFreePhysicalMemorySize(); + long getTotalPhysicalMemorySize(); + } +} diff --git a/spark-common/src/main/java/me/lucko/spark/common/monitor/tick/TickStatistics.java b/spark-common/src/main/java/me/lucko/spark/common/monitor/tick/TickStatistics.java index 31b58e9..bd2b834 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/monitor/tick/TickStatistics.java +++ b/spark-common/src/main/java/me/lucko/spark/common/monitor/tick/TickStatistics.java @@ -56,7 +56,8 @@ public class TickStatistics implements TickHook.Callback, TickReporter.Callback private boolean durationSupported = false; private final RollingAverage tickDuration10Sec = new RollingAverage(TPS * 10); private final RollingAverage tickDuration1Min = new RollingAverage(TPS * 60); - private final RollingAverage[] tickDurationAverages = {this.tickDuration10Sec, this.tickDuration1Min}; + private final RollingAverage tickDuration5Min = new RollingAverage(TPS * 60 * 5); + private final RollingAverage[] tickDurationAverages = {this.tickDuration10Sec, this.tickDuration1Min, this.tickDuration5Min}; private long last = 0; @@ -131,6 +132,13 @@ public class TickStatistics implements TickHook.Callback, TickReporter.Callback return this.tickDuration1Min; } + public RollingAverage duration5Min() { + if (!this.durationSupported) { + return null; + } + return this.tickDuration5Min; + } + /** * Rolling average calculator. diff --git a/spark-common/src/main/java/me/lucko/spark/common/platform/AbstractPlatformInfo.java b/spark-common/src/main/java/me/lucko/spark/common/platform/AbstractPlatformInfo.java deleted file mode 100644 index 645d5b2..0000000 --- a/spark-common/src/main/java/me/lucko/spark/common/platform/AbstractPlatformInfo.java +++ /dev/null @@ -1,17 +0,0 @@ -package me.lucko.spark.common.platform; - -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryUsage; - -public abstract class AbstractPlatformInfo implements PlatformInfo { - - @Override - public int getNCpus() { - return Runtime.getRuntime().availableProcessors(); - } - - @Override - public MemoryUsage getHeapUsage() { - return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - } -} diff --git a/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformInfo.java b/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformInfo.java index 80fb85f..eb1b25d 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformInfo.java +++ b/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformInfo.java @@ -20,13 +20,12 @@ package me.lucko.spark.common.platform; -import me.lucko.spark.proto.SparkProtos; import me.lucko.spark.proto.SparkProtos.PlatformMetadata; -import java.lang.management.MemoryUsage; - public interface PlatformInfo { + int DATA_VERSION = 1; + Type getType(); String getName(); @@ -35,18 +34,13 @@ public interface PlatformInfo { String getMinecraftVersion(); - int getNCpus(); - - MemoryUsage getHeapUsage(); - default int getSparkVersion() { // does not necessarily correspond to the plugin/mod version - // this is like a data version I suppose - return 1; + return DATA_VERSION; } default Data toData() { - return new Data(getType(), getName(), getVersion(), getMinecraftVersion(), getNCpus(), getHeapUsage(), getSparkVersion()); + return new Data(getType(), getName(), getVersion(), getMinecraftVersion(), getSparkVersion()); } enum Type { @@ -70,17 +64,13 @@ public interface PlatformInfo { private final String name; private final String version; private final String minecraftVersion; - private final int nCpus; - private final MemoryUsage heapUsage; private final int sparkVersion; - public Data(Type type, String name, String version, String minecraftVersion, int nCpus, MemoryUsage heapUsage, int sparkVersion) { + public Data(Type type, String name, String version, String minecraftVersion, int sparkVersion) { this.type = type; this.name = name; this.version = version; this.minecraftVersion = minecraftVersion; - this.nCpus = nCpus; - this.heapUsage = heapUsage; this.sparkVersion = sparkVersion; } @@ -100,33 +90,15 @@ public interface PlatformInfo { return this.minecraftVersion; } - public int getNCpus() { - return this.nCpus; - } - - public MemoryUsage getHeapUsage() { - return this.heapUsage; - } - public int getSparkVersion() { return this.sparkVersion; } - public SparkProtos.MemoryUsage getHeapUsageProto() { - return SparkProtos.MemoryUsage.newBuilder() - .setUsed(this.heapUsage.getUsed()) - .setCommitted(this.heapUsage.getCommitted()) - .setMax(this.heapUsage.getMax()) - .build(); - } - public PlatformMetadata toProto() { PlatformMetadata.Builder proto = PlatformMetadata.newBuilder() .setType(this.type.toProto()) .setName(this.name) .setVersion(this.version) - .setNCpus(this.nCpus) - .setHeapUsage(getHeapUsageProto()) .setSparkVersion(this.sparkVersion); if (this.minecraftVersion != null) { diff --git a/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformStatisticsProvider.java b/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformStatisticsProvider.java new file mode 100644 index 0000000..5608fcc --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/platform/PlatformStatisticsProvider.java @@ -0,0 +1,165 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) <luck@lucko.me> + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +package me.lucko.spark.common.platform; + +import me.lucko.spark.common.SparkPlatform; +import me.lucko.spark.common.monitor.cpu.CpuMonitor; +import me.lucko.spark.common.monitor.disk.DiskUsage; +import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics; +import me.lucko.spark.common.monitor.memory.MemoryInfo; +import me.lucko.spark.common.monitor.tick.TickStatistics; +import me.lucko.spark.common.util.RollingAverage; +import me.lucko.spark.proto.SparkProtos.PlatformStatistics; +import me.lucko.spark.proto.SparkProtos.SystemStatistics; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; +import java.util.Map; + +public class PlatformStatisticsProvider { + private final SparkPlatform platform; + + public PlatformStatisticsProvider(SparkPlatform platform) { + this.platform = platform; + } + + public SystemStatistics getSystemStatistics() { + SystemStatistics.Builder builder = SystemStatistics.newBuilder() + .setCpu(SystemStatistics.Cpu.newBuilder() + .setThreads(Runtime.getRuntime().availableProcessors()) + .setProcessUsage(SystemStatistics.Cpu.Usage.newBuilder() + .setLast1M(CpuMonitor.processLoad1MinAvg()) + .setLast15M(CpuMonitor.processLoad15MinAvg()) + .build() + ) + .setSystemUsage(SystemStatistics.Cpu.Usage.newBuilder() + .setLast1M(CpuMonitor.systemLoad1MinAvg()) + .setLast15M(CpuMonitor.systemLoad15MinAvg()) + .build() + ) + .build() + ) + .setMemory(SystemStatistics.Memory.newBuilder() + .setPhysical(SystemStatistics.Memory.MemoryPool.newBuilder() + .setUsed(MemoryInfo.getUsedPhysicalMemory()) + .setTotal(MemoryInfo.getTotalPhysicalMemory()) + .build() + ) + .setSwap(SystemStatistics.Memory.MemoryPool.newBuilder() + .setUsed(MemoryInfo.getUsedSwap()) + .setTotal(MemoryInfo.getTotalSwap()) + .build() + ) + .build() + ) + .setDisk(SystemStatistics.Disk.newBuilder() + .setTotal(DiskUsage.getTotal()) + .setUsed(DiskUsage.getUsed()) + .build() + ) + .setOs(SystemStatistics.Os.newBuilder() + .setArch(System.getProperty("os.arch")) + .setName(System.getProperty("os.name")) + .setVersion(System.getProperty("os.version")) + .build() + ) + .setJava(SystemStatistics.Java.newBuilder() + .setVendor(System.getProperty("java.vendor")) + .setVersion(System.getProperty("java.version")) + .setVendorVersion(System.getProperty("java.vendor.version")) + .build() + ); + + long uptime = ManagementFactory.getRuntimeMXBean().getUptime(); + builder.setUptime(uptime); + + Map<String, GarbageCollectorStatistics> gcStats = GarbageCollectorStatistics.pollStats(); + gcStats.forEach((name, statistics) -> builder.putGc( + name, + SystemStatistics.Gc.newBuilder() + .setTotal(statistics.getCollectionCount()) + .setAvgTime(statistics.getAverageCollectionTime()) + .setAvgFrequency(statistics.getAverageCollectionFrequency(uptime)) + .build() + )); + + return builder.build(); + } + + public PlatformStatistics getPlatformStatistics(Map<String, GarbageCollectorStatistics> startingGcStatistics) { + PlatformStatistics.Builder builder = PlatformStatistics.newBuilder(); + + MemoryUsage memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + builder.setMemory(PlatformStatistics.Memory.newBuilder() + .setHeap(PlatformStatistics.Memory.MemoryPool.newBuilder() + .setUsed(memoryUsage.getUsed()) + .setTotal(memoryUsage.getCommitted()) + .build() + ) + .build() + ); + + long uptime = System.currentTimeMillis() - this.platform.getServerNormalOperationStartTime(); + builder.setUptime(uptime); + + if (startingGcStatistics != null) { + Map<String, GarbageCollectorStatistics> gcStats = GarbageCollectorStatistics.pollStatsSubtractInitial(startingGcStatistics); + gcStats.forEach((name, statistics) -> builder.putGc( + name, + PlatformStatistics.Gc.newBuilder() + .setTotal(statistics.getCollectionCount()) + .setAvgTime(statistics.getAverageCollectionTime()) + .setAvgFrequency(statistics.getAverageCollectionFrequency(uptime)) + .build() + )); + } + + TickStatistics tickStatistics = this.platform.getTickStatistics(); + if (tickStatistics != null) { + builder.setTps(PlatformStatistics.Tps.newBuilder() + .setLast1M(tickStatistics.tps1Min()) + .setLast5M(tickStatistics.tps5Min()) + .setLast15M(tickStatistics.tps15Min()) + .build() + ); + if (tickStatistics.isDurationSupported()) { + builder.setMspt(PlatformStatistics.Mspt.newBuilder() + .setLast1M(msptValues(tickStatistics.duration1Min())) + .setLast5M(msptValues(tickStatistics.duration5Min())) + .build() + ); + } + } + + return builder.build(); + } + + private static PlatformStatistics.Mspt.Values msptValues(RollingAverage rollingAverage) { + return PlatformStatistics.Mspt.Values.newBuilder() + .setMean(rollingAverage.mean()) + .setMax(rollingAverage.max()) + .setMin(rollingAverage.min()) + .setMedian(rollingAverage.median()) + .setPercentile95(rollingAverage.percentile95th()) + .build(); + } + +} diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java index bae93b1..568b59d 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/AbstractSampler.java @@ -20,6 +20,9 @@ package me.lucko.spark.common.sampler; +import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics; + +import java.util.Map; import java.util.concurrent.CompletableFuture; /** @@ -42,6 +45,9 @@ public abstract class AbstractSampler implements Sampler { /** A future to encapsulate the completion of this sampler instance */ protected final CompletableFuture<Sampler> future = new CompletableFuture<>(); + /** The garbage collector statistics when profiling started */ + protected Map<String, GarbageCollectorStatistics> initialGcStats; + protected AbstractSampler(int interval, ThreadDumper threadDumper, long endTime) { this.interval = interval; this.threadDumper = threadDumper; @@ -65,4 +71,12 @@ public abstract class AbstractSampler implements Sampler { public CompletableFuture<Sampler> getFuture() { return this.future; } + + protected void recordInitialGcStats() { + this.initialGcStats = GarbageCollectorStatistics.pollStats(); + } + + protected Map<String, GarbageCollectorStatistics> getInitialGcStats() { + return this.initialGcStats; + } } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java index b71aaee..d27b2fc 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/Sampler.java @@ -20,12 +20,12 @@ package me.lucko.spark.common.sampler; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.util.ClassSourceLookup; -import me.lucko.spark.proto.SparkProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; import java.util.Comparator; import java.util.Map; @@ -68,6 +68,6 @@ public interface Sampler { CompletableFuture<Sampler> getFuture(); // Methods used to export the sampler data to the web viewer. - SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup); + SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup); } diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java index e99114a..5cc41b9 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadDumper.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler; import me.lucko.spark.common.util.ThreadFinder; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java index e63ebc8..225f768 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/ThreadGrouper.java @@ -20,7 +20,7 @@ package me.lucko.spark.common.sampler; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java index 8b90639..3b1d349 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/aggregator/DataAggregator.java @@ -21,7 +21,7 @@ package me.lucko.spark.common.sampler.aggregator; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.util.Map; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java index 594d56e..3de3943 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncDataAggregator.java @@ -24,7 +24,7 @@ import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.AbstractDataAggregator; import me.lucko.spark.common.sampler.node.StackTraceNode; import me.lucko.spark.common.sampler.node.ThreadNode; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; /** * Data aggregator for {@link AsyncSampler}. diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java index f1d7209..06db795 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncProfilerAccess.java @@ -102,7 +102,7 @@ public enum AsyncProfilerAccess { } // extract the profiler binary from the spark jar file - String resource = os + "/libasyncProfiler.so"; + String resource = "spark/" + os + "/libasyncProfiler.so"; URL profilerResource = AsyncProfilerAccess.class.getClassLoader().getResource(resource); if (profilerResource == null) { throw new IllegalStateException("Could not find " + resource + " in spark jar file"); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java index 1837cbc..db8802c 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/async/AsyncSampler.java @@ -22,8 +22,8 @@ package me.lucko.spark.common.sampler.async; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.AbstractSampler; import me.lucko.spark.common.sampler.ThreadDumper; import me.lucko.spark.common.sampler.ThreadGrouper; @@ -32,7 +32,8 @@ import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.util.ClassSourceLookup; import me.lucko.spark.common.util.TemporaryFiles; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import one.profiler.AsyncProfiler; @@ -117,6 +118,7 @@ public class AsyncSampler extends AbstractSampler { } } + recordInitialGcStats(); scheduleTimeout(); } @@ -154,9 +156,11 @@ public class AsyncSampler extends AbstractSampler { } @Override - public SparkProtos.SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { - final SparkProtos.SamplerMetadata.Builder metadata = SparkProtos.SamplerMetadata.newBuilder() - .setPlatformMetadata(platformInfo.toData().toProto()) + public SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { + final SamplerMetadata.Builder metadata = SamplerMetadata.newBuilder() + .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) + .setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(getInitialGcStats())) + .setSystemStatistics(platform.getStatisticsProvider().getSystemStatistics()) .setCreator(creator.toData().toProto()) .setStartTime(this.startTime) .setInterval(this.interval) @@ -167,7 +171,7 @@ public class AsyncSampler extends AbstractSampler { metadata.setComment(comment); } - SparkProtos.SamplerData.Builder proto = SparkProtos.SamplerData.newBuilder(); + SamplerData.Builder proto = SamplerData.newBuilder(); proto.setMetadata(metadata.build()); aggregateOutput(); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java index 02d5f01..c873f9f 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/JavaSampler.java @@ -23,8 +23,8 @@ package me.lucko.spark.common.sampler.java; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import me.lucko.spark.common.SparkPlatform; import me.lucko.spark.common.command.sender.CommandSender; -import me.lucko.spark.common.platform.PlatformInfo; import me.lucko.spark.common.sampler.AbstractSampler; import me.lucko.spark.common.sampler.ThreadDumper; import me.lucko.spark.common.sampler.ThreadGrouper; @@ -32,8 +32,8 @@ import me.lucko.spark.common.sampler.node.MergeMode; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.tick.TickHook; import me.lucko.spark.common.util.ClassSourceLookup; -import me.lucko.spark.proto.SparkProtos.SamplerData; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerData; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -129,9 +129,11 @@ public class JavaSampler extends AbstractSampler implements Runnable { } @Override - public SamplerData toProto(PlatformInfo platformInfo, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { + public SamplerData toProto(SparkPlatform platform, CommandSender creator, Comparator<? super Map.Entry<String, ThreadNode>> outputOrder, String comment, MergeMode mergeMode, ClassSourceLookup classSourceLookup) { final SamplerMetadata.Builder metadata = SamplerMetadata.newBuilder() - .setPlatformMetadata(platformInfo.toData().toProto()) + .setPlatformMetadata(platform.getPlugin().getPlatformInfo().toData().toProto()) + .setPlatformStatistics(platform.getStatisticsProvider().getPlatformStatistics(getInitialGcStats())) + .setSystemStatistics(platform.getStatisticsProvider().getSystemStatistics()) .setCreator(creator.toData().toProto()) .setStartTime(this.startTime) .setInterval(this.interval) diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java index e7113a1..39e21aa 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/SimpleDataAggregator.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler.java; import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.DataAggregator; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.util.concurrent.ExecutorService; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java index 018a3b8..ac34d01 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/java/TickedDataAggregator.java @@ -24,7 +24,7 @@ import me.lucko.spark.common.sampler.ThreadGrouper; import me.lucko.spark.common.sampler.aggregator.DataAggregator; import me.lucko.spark.common.sampler.node.ThreadNode; import me.lucko.spark.common.tick.TickHook; -import me.lucko.spark.proto.SparkProtos.SamplerMetadata; +import me.lucko.spark.proto.SparkSamplerProtos.SamplerMetadata; import java.lang.management.ThreadInfo; import java.util.ArrayList; diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java index f935fb2..54217be 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/StackTraceNode.java @@ -22,7 +22,7 @@ package me.lucko.spark.common.sampler.node; import me.lucko.spark.common.util.MethodDisambiguator; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos; import org.checkerframework.checker.nullness.qual.Nullable; @@ -65,8 +65,8 @@ public final class StackTraceNode extends AbstractNode implements Comparable<Sta return this.description.parentLineNumber; } - public SparkProtos.StackTraceNode toProto(MergeMode mergeMode) { - SparkProtos.StackTraceNode.Builder proto = SparkProtos.StackTraceNode.newBuilder() + public SparkSamplerProtos.StackTraceNode toProto(MergeMode mergeMode) { + SparkSamplerProtos.StackTraceNode.Builder proto = SparkSamplerProtos.StackTraceNode.newBuilder() .setTime(getTotalTime()) .setClassName(this.description.className) .setMethodName(this.description.methodName); diff --git a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/ThreadNode.java b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/ThreadNode.java index 5cac33d..fc56987 100644 --- a/spark-common/src/main/java/me/lucko/spark/common/sampler/node/ThreadNode.java +++ b/spark-common/src/main/java/me/lucko/spark/common/sampler/node/ThreadNode.java @@ -20,7 +20,7 @@ package me.lucko.spark.common.sampler.node; -import me.lucko.spark.proto.SparkProtos; +import me.lucko.spark.proto.SparkSamplerProtos; /** * The root of a sampling stack for a given thread / thread group. @@ -36,8 +36,8 @@ public final class ThreadNode extends AbstractNode { this.threadName = threadName; } - public SparkProtos.ThreadNode toProto(MergeMode mergeMode) { - SparkProtos.ThreadNode.Builder proto = SparkProtos.ThreadNode.newBuilder() + public SparkSamplerProtos.ThreadNode toProto(MergeMode mergeMode) { + SparkSamplerProtos.ThreadNode.Builder proto = SparkSamplerProtos.ThreadNode.newBuilder() .setName(this.threadName) .setTime(getTotalTime()); diff --git a/spark-common/src/main/java/me/lucko/spark/common/util/Compression.java b/spark-common/src/main/java/me/lucko/spark/common/util/Compression.java new file mode 100644 index 0000000..9295c25 --- /dev/null +++ b/spark-common/src/main/java/me/lucko/spark/common/util/Compression.java @@ -0,0 +1,100 @@ +/* + * This file is part of spark. + * + * Copyright (c) lucko (Luck) <luck@lucko.me> + * Copyright (c) contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +package me.lucko.spark.common.util; + +import org.tukaani.xz.LZMA2Options; +import org.tukaani.xz.LZMAOutputStream; +import org.tukaani.xz.XZOutputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.LongConsumer; +import java.util.zip.GZIPOutputStream; + +public enum Compression { + GZIP { + @Override + public Path compress(Path file, LongConsumer progressHandler) throws IOException { + Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".gz"); + try (InputStream in = Files.newInputStream(file)) { + try (OutputStream out = Files.newOutputStream(compressedFile)) { + try (GZIPOutputStream compressionOut = new GZIPOutputStream(out, 1024 * 64)) { + copy(in, compressionOut, progressHandler); + } + } + } + return compressedFile; + } + }, + XZ { + @Override + public Path compress(Path file, LongConsumer progressHandler) throws IOException { + Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".xz"); + try (InputStream in = Files.newInputStream(file)) { + try (OutputStream out = Files.newOutputStream(compressedFile)) { + try (XZOutputStream compressionOut = new XZOutputStream(out, new LZMA2Options())) { + copy(in, compressionOut, progressHandler); + } + } + } + return compressedFile; + } + }, + LZMA { + @Override + public Path compress(Path file, LongConsumer progressHandler) throws IOException { + Path compressedFile = file.getParent().resolve(file.getFileName().toString() + ".lzma"); + try (InputStream in = Files.newInputStream(file)) { + try (OutputStream out = Files.newOutputStream(compressedFile)) { + try (LZMAOutputStream compressionOut = new LZMAOutputStream(out, new LZMA2Options(), true)) { + copy(in, compressionOut, progressHandler); + } + } + } + return compressedFile; + } + }; + + public abstract Path compress(Path file, LongConsumer progressHandler) throws IOException; + + private static long copy(InputStream from, OutputStream to, LongConsumer progress) throws IOException { + byte[] buf = new byte[1024 * 64]; + long total = 0; + long iterations = 0; + while (true) { + int r = from.read(buf); + if (r == -1) { + break; + } + to.write(buf, 0, r); + total += r; + + // report progress every 5MB + if (iterations++ % ((1024 / 64) * 5) == 0) { + progress.accept(total); + } + } + return total; + } +} |