andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add histogram plots

Browse Source
This commit is contained in:
Andreas Huber
2019-12-27 12:25:25 +01:00
parent 2268ab40b3
commit 19e6dd1102
8 changed files with 207 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@@ -22,7 +22,7 @@ ext {
javaVersion=12
version_log4j2= '2.13.0'
version_log4j2= '2.12.1'
version_spring = '2.2.2.RELEASE'
lib_antlr = "org.antlr:antlr4:4.7.2"

6
pdb-js/src/app/plot.service.ts
View File

@@ -17,7 +17,7 @@ export class PlotService {
this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration));
this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration));
this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", false, DataType.Group, DataType.Duration));
this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount));
this.plotTypes.push(new PlotType("RIDGELINES", "Ridgelines", "ridgelines", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("QQ", "Quantile-Quantile", "quantile-quantile", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count));
@@ -26,6 +26,8 @@ export class PlotService {
this.plotTypes.push(new PlotType("PIE", "Pie", "pie-chart", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("STEP_FIT", "Step Fit", "step-fit", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("LAG", "Lag", "lag-plot", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("ACF", "ACF", "acf-plot", false, DataType.Other, DataType.Other));
}
ngOnInit() {
@@ -125,6 +127,8 @@ export enum DataType {
Count,
Group,
Metric,
HistogramBin,
HistogramCount,
Other
}

2
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java
View File

@@ -15,4 +15,6 @@ public enum Aggregate {
* @see https://serialmentor.com/dataviz/ecdf-qq.html
*/
CUM_DISTRIBUTION,
HISTOGRAM
}

101
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java Normal file
View File

@@ -0,0 +1,101 @@
package org.lucares.pdb.plot.api;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
public class HistogramAggregator implements CustomAggregator {
private final static class ToBins implements LongLongConsumer {
final LongLongHashMap bins;
private final int binWidth;
public ToBins(final int numBins, final int binWidth) {
this.binWidth = binWidth;
// use large initial capacity to reduce likelihood of collisions in hashmap
bins = new LongLongHashMap(numBins * 5, 0.75);
}
@Override
public void accept(final long key, final long value) {
final long bin = binWidth * (key / binWidth);
bins.compute(bin, 0, oldValue -> oldValue + value);
}
public LongLongHashMap getBins() {
return bins;
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long min;
private long max;
private final Path tmpDir;
private final PlotSettings plotSettings;
public HistogramAggregator(final Path tmpDir, final PlotSettings plotSettings) {
this.tmpDir = tmpDir;
this.plotSettings = plotSettings;
}
@Override
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
map.compute(value, 0, l -> l + 1);
min = min < value ? min : value;
max = max > value ? max : value;
}
@Override
public AggregatedData getAggregatedData() throws IOException {
final char separator = ',';
final char newline = '\n';
final int numBins = plotSettings.getWidth() / 3;
final int binWidth = Math.max((int) (max) / numBins, 1);
final ToBins toBins = new ToBins(numBins, binWidth);
map.forEachOrdered(toBins);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (map.size() > 0) {
// compute the percentiles
final LongLongHashMap bins = toBins.getBins();
for (int i = 0; i < numBins; i++) {
final int bin = i * binWidth;
final long value = bins.get(bin, 0);
data.append(bin);
data.append(separator);
data.append(value);
data.append(newline);
}
}
output.write(data.toString());
System.out.println(data.toString());
}
final AggregatedData result = new AggregatedData("histogram", dataFile);
return result;
}
@Override
public Aggregate getType() {
return Aggregate.HISTOGRAM;
}
}

73
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java Normal file
View File

@@ -0,0 +1,73 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
public class HistogramHandler extends AggregateHandler {
@Override
Type getAxisType(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
return Type.HistogramBin;
case Y1:
case Y2:
return Type.HistogramCount;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
@Override
Aggregate getAggregateType() {
return Aggregate.HISTOGRAM;
}
@Override
AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Histogram - Duration in ms");
result.setType(Type.HistogramBin);
result.setAxis(getxAxis());
result.setTicsEnabled(true);
return result;
}
@Override
AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Histogram - Count");
result.setType(Type.HistogramCount);
result.setAxis(getyAxis());
result.setTicsEnabled(true);
result.setFrom("0");
return result;
}
@Override
void addPlot(final StringBuilder result, final AggregatedData aggregatedData, final LineStyle lineStyle,
final Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
gnuplotXYAxis(), //
lineStyle.darker()//
);
}
@Override
CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new HistogramAggregator(tmpDir, plotSettings);
}
}

32
pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java
View File

@@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
public class AxisSettings {
public enum Type {
Number, Time, Duration, Percent
Number, Time, Duration, Percent, HistogramBin, HistogramCount
}
private String format = "";
@@ -38,7 +38,7 @@ public class AxisSettings {
return format;
}
public void setFormat(String format) {
public void setFormat(final String format) {
this.format = format;
}
@@ -46,7 +46,7 @@ public class AxisSettings {
return label;
}
public void setLabel(String label) {
public void setLabel(final String label) {
this.label = label;
}
@@ -54,7 +54,7 @@ public class AxisSettings {
return rotateLabel;
}
public void setRotateLabel(int rotateLabel) {
public void setRotateLabel(final int rotateLabel) {
this.rotateLabel = rotateLabel;
}
@@ -62,7 +62,7 @@ public class AxisSettings {
return from;
}
public void setFrom(String from) {
public void setFrom(final String from) {
this.from = from;
}
@@ -70,7 +70,7 @@ public class AxisSettings {
return to;
}
public void setTo(String to) {
public void setTo(final String to) {
this.to = to;
}
@@ -78,7 +78,7 @@ public class AxisSettings {
return type;
}
public void setType(Type type) {
public void setType(final Type type) {
this.type = type;
}
@@ -86,11 +86,11 @@ public class AxisSettings {
return axis;
}
public void setAxis(GnuplotAxis axis) {
public void setAxis(final GnuplotAxis axis) {
this.axis = axis;
}
public void setTicIncrement(double ticIncrement) {
public void setTicIncrement(final double ticIncrement) {
this.ticIncrement = ticIncrement;
}
@@ -98,7 +98,7 @@ public class AxisSettings {
return ticIncrement;
}
public void setTicsEnabled(boolean ticsEnabled) {
public void setTicsEnabled(final boolean ticsEnabled) {
this.ticsEnabled = ticsEnabled;
}
@@ -106,7 +106,7 @@ public class AxisSettings {
return ticsEnabled;
}
public void setLogscale(boolean logscale) {
public void setLogscale(final boolean logscale) {
this.logscale = logscale;
}
@@ -114,7 +114,7 @@ public class AxisSettings {
return logscale;
}
public void setTics(List<String> ticsLabels) {
public void setTics(final List<String> ticsLabels) {
this.ticsLabels = ticsLabels;
}
@@ -122,8 +122,8 @@ public class AxisSettings {
return ticsLabels;
}
public String toGnuplotDefinition(boolean renderLabels) {
StringBuilder result = new StringBuilder();
public String toGnuplotDefinition(final boolean renderLabels) {
final StringBuilder result = new StringBuilder();
if (type == Type.Time) {
appendfln(result, "set %sdata time", axis);
}
@@ -171,10 +171,10 @@ public class AxisSettings {
@Override
public String toString() {
ObjectMapper mapper = new ObjectMapper();
final ObjectMapper mapper = new ObjectMapper();
try {
return mapper.writeValueAsString(this);
} catch (JsonProcessingException e) {
} catch (final JsonProcessingException e) {
return e.getMessage();
}
}

5
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -24,7 +24,7 @@ public class GnuplotFileGenerator implements Appender {
appendfln(result, "set timefmt '%s'", settings.getTimefmt());
final List<AxisSettings> xAxisDefinitions = settings.getAggregates().getXAxisDefinitions(settings, dataSeries);
for (AxisSettings axisSettings : xAxisDefinitions) {
for (final AxisSettings axisSettings : xAxisDefinitions) {
appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels()));
}
@@ -34,10 +34,9 @@ public class GnuplotFileGenerator implements Appender {
// Workaround is to explicitly specify the y-axis range.
// We choose a range for which no ticks are defined. This creates an empty
// y-axis.
yAxisDefinitions.forEach(s -> s.setFrom("0"));
yAxisDefinitions.forEach(s -> s.setFrom("-1"));
}
for (AxisSettings axisSettings : yAxisDefinitions) {
for (final AxisSettings axisSettings : yAxisDefinitions) {
appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels()));
}

10
pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java
View File

@@ -4,6 +4,7 @@ import org.lucares.pdb.plot.api.Aggregate;
import org.lucares.pdb.plot.api.AggregateHandlerCollection;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.CumulativeDistributionHandler;
import org.lucares.pdb.plot.api.HistogramHandler;
import org.lucares.pdb.plot.api.ParallelRequestsAggregate;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.ScatterAggregateHandler;
@@ -55,11 +56,11 @@ class PlotSettingsTransformer {
throw new IllegalStateException("unhandled enum value: " + yRangeUnit);
}
static AggregateHandlerCollection toAggregateInternal(TimeRangeUnitInternal yRangeUnit, AxisScale yAxisScale,
final Iterable<Aggregate> aggregates) {
static AggregateHandlerCollection toAggregateInternal(final TimeRangeUnitInternal yRangeUnit,
final AxisScale yAxisScale, final Iterable<Aggregate> aggregates) {
final AggregateHandlerCollection aggregateHandlerCollection = new AggregateHandlerCollection();
for (Aggregate aggregate : aggregates) {
for (final Aggregate aggregate : aggregates) {
switch (aggregate) {
case CUM_DISTRIBUTION:
@@ -78,6 +79,9 @@ class PlotSettingsTransformer {
aggregateHandlerCollection.add(new ScatterAggregateHandler());
}
break;
case HISTOGRAM:
aggregateHandlerCollection.add(new HistogramHandler());
break;
default:
throw new IllegalStateException("unhandled enum: " + aggregate);
}