From 19e6dd1102f0c29febf372ece3db6e5c29cbfeee Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 27 Dec 2019 12:25:25 +0100 Subject: [PATCH] add histogram plots --- build.gradle | 2 +- pdb-js/src/app/plot.service.ts | 6 +- .../org/lucares/pdb/plot/api/Aggregate.java | 2 + .../pdb/plot/api/HistogramAggregator.java | 101 ++++++++++++++++++ .../pdb/plot/api/HistogramHandler.java | 73 +++++++++++++ .../lucares/recommind/logs/AxisSettings.java | 32 +++--- .../recommind/logs/GnuplotFileGenerator.java | 5 +- .../pdbui/PlotSettingsTransformer.java | 10 +- 8 files changed, 207 insertions(+), 24 deletions(-) create mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java create mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java diff --git a/build.gradle b/build.gradle index bdaa234..9d0ca33 100644 --- a/build.gradle +++ b/build.gradle @@ -22,7 +22,7 @@ ext { javaVersion=12 - version_log4j2= '2.13.0' + version_log4j2= '2.12.1' version_spring = '2.2.2.RELEASE' lib_antlr = "org.antlr:antlr4:4.7.2" diff --git a/pdb-js/src/app/plot.service.ts b/pdb-js/src/app/plot.service.ts index bf55bf8..896b638 100644 --- a/pdb-js/src/app/plot.service.ts +++ b/pdb-js/src/app/plot.service.ts @@ -17,7 +17,7 @@ export class PlotService { this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration)); this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration)); - this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", false, DataType.Group, DataType.Duration)); + this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount)); this.plotTypes.push(new PlotType("RIDGELINES", "Ridgelines", "ridgelines", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("QQ", "Quantile-Quantile", "quantile-quantile", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count)); @@ -26,6 +26,8 @@ export class PlotService { this.plotTypes.push(new PlotType("PIE", "Pie", "pie-chart", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("STEP_FIT", "Step Fit", "step-fit", false, DataType.Other, DataType.Other)); + this.plotTypes.push(new PlotType("LAG", "Lag", "lag-plot", false, DataType.Other, DataType.Other)); + this.plotTypes.push(new PlotType("ACF", "ACF", "acf-plot", false, DataType.Other, DataType.Other)); } ngOnInit() { @@ -125,6 +127,8 @@ export enum DataType { Count, Group, Metric, + HistogramBin, + HistogramCount, Other } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java index 4baa141..b9cc994 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java @@ -15,4 +15,6 @@ public enum Aggregate { * @see https://serialmentor.com/dataviz/ecdf-qq.html */ CUM_DISTRIBUTION, + + HISTOGRAM } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java new file mode 100644 index 0000000..79ee7c3 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java @@ -0,0 +1,101 @@ +package org.lucares.pdb.plot.api; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import org.lucares.collections.LongLongConsumer; +import org.lucares.collections.LongLongHashMap; + +public class HistogramAggregator implements CustomAggregator { + + private final static class ToBins implements LongLongConsumer { + + final LongLongHashMap bins; + + private final int binWidth; + + public ToBins(final int numBins, final int binWidth) { + this.binWidth = binWidth; + // use large initial capacity to reduce likelihood of collisions in hashmap + bins = new LongLongHashMap(numBins * 5, 0.75); + } + + @Override + public void accept(final long key, final long value) { + final long bin = binWidth * (key / binWidth); + bins.compute(bin, 0, oldValue -> oldValue + value); + } + + public LongLongHashMap getBins() { + return bins; + } + } + + // the rather large initial capacity should prevent too many grow&re-hash phases + private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75); + private long min; + private long max; + private final Path tmpDir; + private final PlotSettings plotSettings; + + public HistogramAggregator(final Path tmpDir, final PlotSettings plotSettings) { + this.tmpDir = tmpDir; + this.plotSettings = plotSettings; + } + + @Override + public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { + map.compute(value, 0, l -> l + 1); + min = min < value ? min : value; + max = max > value ? max : value; + } + + @Override + public AggregatedData getAggregatedData() throws IOException { + final char separator = ','; + final char newline = '\n'; + + final int numBins = plotSettings.getWidth() / 3; + final int binWidth = Math.max((int) (max) / numBins, 1); + + final ToBins toBins = new ToBins(numBins, binWidth); + map.forEachOrdered(toBins); + + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final StringBuilder data = new StringBuilder(); + if (map.size() > 0) { + // compute the percentiles + final LongLongHashMap bins = toBins.getBins(); + for (int i = 0; i < numBins; i++) { + final int bin = i * binWidth; + final long value = bins.get(bin, 0); + data.append(bin); + data.append(separator); + data.append(value); + data.append(newline); + + } + + } + output.write(data.toString()); + System.out.println(data.toString()); + + } + final AggregatedData result = new AggregatedData("histogram", dataFile); + return result; + } + + @Override + public Aggregate getType() { + return Aggregate.HISTOGRAM; + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java new file mode 100644 index 0000000..17c4674 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java @@ -0,0 +1,73 @@ +package org.lucares.pdb.plot.api; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Optional; + +import org.lucares.recommind.logs.AxisSettings; +import org.lucares.recommind.logs.AxisSettings.Type; +import org.lucares.recommind.logs.DataSeries; +import org.lucares.recommind.logs.GnuplotAxis; +import org.lucares.recommind.logs.GnuplotSettings; +import org.lucares.recommind.logs.LineStyle; + +public class HistogramHandler extends AggregateHandler { + + @Override + Type getAxisType(final GnuplotAxis axis) { + switch (axis) { + case X1: + case X2: + return Type.HistogramBin; + case Y1: + case Y2: + return Type.HistogramCount; + default: + throw new IllegalArgumentException("Unexpected value: " + axis); + } + } + + @Override + Aggregate getAggregateType() { + return Aggregate.HISTOGRAM; + } + + @Override + AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = new AxisSettings(); + result.setLabel("Histogram - Duration in ms"); + result.setType(Type.HistogramBin); + result.setAxis(getxAxis()); + result.setTicsEnabled(true); + return result; + } + + @Override + AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = new AxisSettings(); + result.setLabel("Histogram - Count"); + result.setType(Type.HistogramCount); + result.setAxis(getyAxis()); + result.setTicsEnabled(true); + result.setFrom("0"); + return result; + } + + @Override + void addPlot(final StringBuilder result, final AggregatedData aggregatedData, final LineStyle lineStyle, + final Optional title) { + appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", // + aggregatedData.getDataFile().getAbsolutePath(), // + gnuplotTitle(title), // + gnuplotXYAxis(), // + lineStyle.darker()// + ); + } + + @Override + CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings, + final long fromEpochMilli, final long toEpochMilli) { + return new HistogramAggregator(tmpDir, plotSettings); + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java index 99139ef..7ca9c4d 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; public class AxisSettings { public enum Type { - Number, Time, Duration, Percent + Number, Time, Duration, Percent, HistogramBin, HistogramCount } private String format = ""; @@ -38,7 +38,7 @@ public class AxisSettings { return format; } - public void setFormat(String format) { + public void setFormat(final String format) { this.format = format; } @@ -46,7 +46,7 @@ public class AxisSettings { return label; } - public void setLabel(String label) { + public void setLabel(final String label) { this.label = label; } @@ -54,7 +54,7 @@ public class AxisSettings { return rotateLabel; } - public void setRotateLabel(int rotateLabel) { + public void setRotateLabel(final int rotateLabel) { this.rotateLabel = rotateLabel; } @@ -62,7 +62,7 @@ public class AxisSettings { return from; } - public void setFrom(String from) { + public void setFrom(final String from) { this.from = from; } @@ -70,7 +70,7 @@ public class AxisSettings { return to; } - public void setTo(String to) { + public void setTo(final String to) { this.to = to; } @@ -78,7 +78,7 @@ public class AxisSettings { return type; } - public void setType(Type type) { + public void setType(final Type type) { this.type = type; } @@ -86,11 +86,11 @@ public class AxisSettings { return axis; } - public void setAxis(GnuplotAxis axis) { + public void setAxis(final GnuplotAxis axis) { this.axis = axis; } - public void setTicIncrement(double ticIncrement) { + public void setTicIncrement(final double ticIncrement) { this.ticIncrement = ticIncrement; } @@ -98,7 +98,7 @@ public class AxisSettings { return ticIncrement; } - public void setTicsEnabled(boolean ticsEnabled) { + public void setTicsEnabled(final boolean ticsEnabled) { this.ticsEnabled = ticsEnabled; } @@ -106,7 +106,7 @@ public class AxisSettings { return ticsEnabled; } - public void setLogscale(boolean logscale) { + public void setLogscale(final boolean logscale) { this.logscale = logscale; } @@ -114,7 +114,7 @@ public class AxisSettings { return logscale; } - public void setTics(List ticsLabels) { + public void setTics(final List ticsLabels) { this.ticsLabels = ticsLabels; } @@ -122,8 +122,8 @@ public class AxisSettings { return ticsLabels; } - public String toGnuplotDefinition(boolean renderLabels) { - StringBuilder result = new StringBuilder(); + public String toGnuplotDefinition(final boolean renderLabels) { + final StringBuilder result = new StringBuilder(); if (type == Type.Time) { appendfln(result, "set %sdata time", axis); } @@ -171,10 +171,10 @@ public class AxisSettings { @Override public String toString() { - ObjectMapper mapper = new ObjectMapper(); + final ObjectMapper mapper = new ObjectMapper(); try { return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { + } catch (final JsonProcessingException e) { return e.getMessage(); } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java index e1075c6..f5f97bb 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -24,7 +24,7 @@ public class GnuplotFileGenerator implements Appender { appendfln(result, "set timefmt '%s'", settings.getTimefmt()); final List xAxisDefinitions = settings.getAggregates().getXAxisDefinitions(settings, dataSeries); - for (AxisSettings axisSettings : xAxisDefinitions) { + for (final AxisSettings axisSettings : xAxisDefinitions) { appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels())); } @@ -34,10 +34,9 @@ public class GnuplotFileGenerator implements Appender { // Workaround is to explicitly specify the y-axis range. // We choose a range for which no ticks are defined. This creates an empty // y-axis. - yAxisDefinitions.forEach(s -> s.setFrom("0")); yAxisDefinitions.forEach(s -> s.setFrom("-1")); } - for (AxisSettings axisSettings : yAxisDefinitions) { + for (final AxisSettings axisSettings : yAxisDefinitions) { appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels())); } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 958f081..7d889a7 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -4,6 +4,7 @@ import org.lucares.pdb.plot.api.Aggregate; import org.lucares.pdb.plot.api.AggregateHandlerCollection; import org.lucares.pdb.plot.api.AxisScale; import org.lucares.pdb.plot.api.CumulativeDistributionHandler; +import org.lucares.pdb.plot.api.HistogramHandler; import org.lucares.pdb.plot.api.ParallelRequestsAggregate; import org.lucares.pdb.plot.api.PlotSettings; import org.lucares.pdb.plot.api.ScatterAggregateHandler; @@ -55,11 +56,11 @@ class PlotSettingsTransformer { throw new IllegalStateException("unhandled enum value: " + yRangeUnit); } - static AggregateHandlerCollection toAggregateInternal(TimeRangeUnitInternal yRangeUnit, AxisScale yAxisScale, - final Iterable aggregates) { + static AggregateHandlerCollection toAggregateInternal(final TimeRangeUnitInternal yRangeUnit, + final AxisScale yAxisScale, final Iterable aggregates) { final AggregateHandlerCollection aggregateHandlerCollection = new AggregateHandlerCollection(); - for (Aggregate aggregate : aggregates) { + for (final Aggregate aggregate : aggregates) { switch (aggregate) { case CUM_DISTRIBUTION: @@ -78,6 +79,9 @@ class PlotSettingsTransformer { aggregateHandlerCollection.add(new ScatterAggregateHandler()); } break; + case HISTOGRAM: + aggregateHandlerCollection.add(new HistogramHandler()); + break; default: throw new IllegalStateException("unhandled enum: " + aggregate); }