From 8f15aba0d58308fa6d8e7e4e44cbdd4fedd8eb63 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sun, 21 Jan 2018 10:54:13 +0100 Subject: [PATCH] replace individual percentile aggregates with a single one for all --- .../pdb/plot/api/AggregateHandler.java | 3 +- .../lucares/pdb/plot/api/AggregatedData.java | 28 +++---- .../pdb/plot/api/CustomAggregator.java | 4 +- .../lucares/pdb/plot/api/MeanAggregate.java | 38 ---------- .../lucares/pdb/plot/api/NullAggregate.java | 7 +- .../pdb/plot/api/PercentileAggregate.java | 42 ++++------ .../plot/api/PercentileCustomAggregator.java | 76 +++++++++++-------- .../recommind/logs/GnuplotFileGenerator.java | 12 +++ .../lucares/recommind/logs/ScatterPlot.java | 2 +- .../pdbui/PlotSettingsTransformer.java | 8 +- .../org/lucares/pdbui/domain/Aggregate.java | 2 +- pdb-ui/src/main/resources/templates/main.html | 9 +-- 12 files changed, 99 insertions(+), 132 deletions(-) delete mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java index 4c97833..8bf835f 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java @@ -1,5 +1,6 @@ package org.lucares.pdb.plot.api; +import java.nio.file.Path; import java.util.Collection; import org.lucares.recommind.logs.DataSeries; @@ -16,6 +17,6 @@ public interface AggregateHandler { void addPlots(StringBuilder result, Collection dataSeries); - CustomAggregator createCustomAggregator(long minDate, long maxDate); + CustomAggregator createCustomAggregator(Path tmpDir); } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java index 51b8898..df8022e 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java @@ -1,21 +1,21 @@ package org.lucares.pdb.plot.api; -import java.util.List; +import java.io.File; public class AggregatedData { -private final String label; -private final List data; -public AggregatedData(String label, List data) { - super(); - this.label = label; - this.data = data; -} -public String getLabel() { - return label; -} -public List getData() { - return data; -} + private final String label; + private File dataFile; + public AggregatedData(String label, File dataFile) { + this.label = label; + this.dataFile = dataFile; + } + public String getLabel() { + return label; + } + + public File getDataFile() { + return dataFile; + } } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java index be0a8e5..074189b 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java @@ -1,9 +1,11 @@ package org.lucares.pdb.plot.api; +import java.io.IOException; + public interface CustomAggregator { void addValue(long epochMilli, long value); - AggregatedData getAggregatedData(); + AggregatedData getAggregatedData() throws IOException; } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java deleted file mode 100644 index 00c9ed8..0000000 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.lucares.pdb.plot.api; - -import java.util.Collection; - -import org.lucares.recommind.logs.DataSeries; -import org.lucares.recommind.logs.FileBackedDataSeries; - -public class MeanAggregate implements AggregateHandler{ - - @Override - public void addGnuplotDefinitions(StringBuilder result, String separator, Collection dataSeries) { - int count = 1; - for (final DataSeries dataSerie : dataSeries) { - if (dataSerie instanceof FileBackedDataSeries){ - appendfln(result, "stats '%s' using 2 prefix \"A%d\"", ((FileBackedDataSeries) dataSerie).getDataFile(),count); - count++; - } - } - } - - @Override - public void addPlots(StringBuilder result, Collection dataSeries) { - int count = 1; - for (final DataSeries dataSerie : dataSeries) { - if (dataSerie instanceof FileBackedDataSeries){ - appendfln(result, "A%d_mean title '%s Mean', \\", count, - dataSerie.getTitle(), dataSerie.getTitle()); - count++; - } - } - } - - @Override - public CustomAggregator createCustomAggregator(long minDate, long maxDate) { - return new NullCustomAggregator(); - } - -} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java index 6304d6e..0c97457 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java @@ -1,14 +1,15 @@ package org.lucares.pdb.plot.api; +import java.nio.file.Path; import java.util.Collection; import org.lucares.recommind.logs.DataSeries; public class NullAggregate implements AggregateHandler { - @Override - public void addGnuplotDefinitions(StringBuilder result, String separator ,Collection dataSeries) { + public void addGnuplotDefinitions(StringBuilder result, String separator, + Collection dataSeries) { // nothing to do; this is a Null-Object } @@ -18,7 +19,7 @@ public class NullAggregate implements AggregateHandler { } @Override - public CustomAggregator createCustomAggregator(long minDate, long maxDate) { + public CustomAggregator createCustomAggregator(Path tmpDir) { return new NullCustomAggregator(); } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java index e9c3f81..fc6f24c 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java @@ -1,52 +1,38 @@ package org.lucares.pdb.plot.api; +import java.nio.file.Path; import java.util.Collection; import org.lucares.recommind.logs.DataSeries; public class PercentileAggregate implements AggregateHandler{ - private double percentile; - - /** - * - * @param percentile range > 0.0 and < 1.0) - */ - public PercentileAggregate(double percentile) { - this.percentile = percentile; + @Override + public CustomAggregator createCustomAggregator(Path tmpDir) { + return new PercentileCustomAggregator(tmpDir); + } + + + public PercentileAggregate() { } @Override public void addGnuplotDefinitions(StringBuilder result, String separator, Collection dataSeries) { - for (DataSeries dataSerie : dataSeries) { - if (dataSerie.getAggregatedData() != null){ - - appendfln(result, "$%s << EOD", dataSerie.getId()); - - for (AggregatedDataEntry aggregatedData : dataSerie.getAggregatedData().getData()) { - appendfln(result, "%.3f%s%d", aggregatedData.getEpochSeconds(), separator, aggregatedData.getValue()); - } - - - appendfln(result, "EOD", dataSerie.getId()); - } - } + // TODO still neeeded? } @Override public void addPlots(StringBuilder result, Collection dataSeries) { for (DataSeries dataSerie : dataSeries) { - if (dataSerie.getAggregatedData() != null){ - appendfln(result, "'$%s' using 1:2 title '%s' with linespoints lw 2, \\", dataSerie.getId(), dataSerie.getTitle()+" " +dataSerie.getAggregatedData().getLabel()); + final AggregatedData aggregatedData = dataSerie.getAggregatedData(); + if (aggregatedData != null){ + appendfln(result, "'%s' using 1:2 title '%s' with lines lw 1 axes x2y1, \\", // + aggregatedData.getDataFile().getAbsolutePath(),// + dataSerie.getTitle()+" " +aggregatedData.getLabel()); } } } - @Override - public CustomAggregator createCustomAggregator(long minDate, long maxDate) { - return new PercentileCustomAggregator(percentile, minDate, maxDate); - } - } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java index 5be09ba..7de8afd 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java @@ -1,54 +1,68 @@ package org.lucares.pdb.plot.api; -import java.util.ArrayList; -import java.util.List; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import org.lucares.collections.IntList; public class PercentileCustomAggregator implements CustomAggregator{ - private double percentile; - private long fromEpochMilli; - private long numBuckets; - private long bucketSize; + private final static int POINTS = 300; - private List buckets = new ArrayList<>(); + private final IntList values = new IntList(); // TODO should be a LongList - public PercentileCustomAggregator(double percentile, long fromEpochMilli, long toEpochMilli) { - this.percentile = percentile; - this.fromEpochMilli = fromEpochMilli; - this.numBuckets = 50; - this.bucketSize = (toEpochMilli - fromEpochMilli) / numBuckets; - for(int i = 0; i < numBuckets; i++){ - buckets.add(new IntList()); - } + private Path tmpDir; + + + + public PercentileCustomAggregator(Path tmpDir) { + this.tmpDir = tmpDir; } @Override public void addValue(long epochMilli, long value) { - final int bucket = (int) ((epochMilli - fromEpochMilli ) / bucketSize); - buckets.get(bucket).add((int)value); // TODO we should use a LongList instead of an IntList + values.add((int) value); } @Override - public AggregatedData getAggregatedData() { - final List data = new ArrayList<>((int)numBuckets); + public AggregatedData getAggregatedData() throws IOException { + final char separator = ','; + final char newline = '\n'; - for (int i= 0; i < numBuckets; i++) { - final IntList values = buckets.get(i); - values.sort(); - final int indexOfPercentile = (int) (values.size() * percentile); - final long value = values.size() > indexOfPercentile ? values.get(indexOfPercentile) : 0; - final long epochMilli = fromEpochMilli + i*bucketSize + bucketSize/2; // the middle of the bucket - final double epochSecond = epochMilli / 1000.0; // gnuplot wants the time in seconds - final AggregatedDataEntry aggregatedDataEntry = new AggregatedDataEntry(epochSecond, value); + values.parallelSort(); + + + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){ - if (value > 0){ - data.add(aggregatedDataEntry ); + final StringBuilder data = new StringBuilder(); + if (values.size() > 0) { + // compute the percentiles + for (int i = 0; i < POINTS; i++) { + data.append(i* (100/(double)POINTS)); + data.append(separator); + data.append(values.get((int) Math.floor(values.size() + / ((double)POINTS) * i))); + data.append(newline); + } + final int maxValue = values.get(values.size() - 1); + data.append(100); + data.append(separator); + data.append(maxValue); + data.append(newline); } + output.write(data.toString()); } - final String title = String.format("%.1f%% percentile", percentile*100); - return new AggregatedData(title, data); + + + final String title = String.format("percentiles"); + return new AggregatedData(title, dataFile); } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java index ea2ee7e..f67b50a 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -30,6 +30,18 @@ public class GnuplotFileGenerator { appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel()); appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo()); + /* + * set yrange ["0":"100"] + set ytics nomirror # don't show the tics of the left y-axis on the right side + + set y2label "CPU load" + set y2tics + set autoscale y2 + */ + appendfln(result, "set x2label \"percent\""); + appendfln(result, "set y2tics"); + appendfln(result, "set x2range [\"0\":\"100\"]"); + final long graphOffset = settings.getYAxisScale() == AxisScale.LINEAR ? 0 : 1; appendfln(result, "set yrange [\""+graphOffset+"\":]"); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java index 6a8ca6c..c947da4 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java @@ -171,7 +171,7 @@ public class ScatterPlot implements ConcretePlotter { final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); - final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli); + final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir); long maxValue = 0; long ignoredValues = 0; diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 9aa51d8..17a6b97 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -1,7 +1,6 @@ package org.lucares.pdbui; import org.lucares.pdb.plot.api.AggregateHandler; -import org.lucares.pdb.plot.api.MeanAggregate; import org.lucares.pdb.plot.api.NullAggregate; import org.lucares.pdb.plot.api.PercentileAggregate; import org.lucares.pdb.plot.api.PlotSettings; @@ -32,12 +31,7 @@ class PlotSettingsTransformer { private static AggregateHandler toAggregateInternal(Aggregate aggregate) { switch (aggregate) { case NONE:return new NullAggregate(); - case MEAN:return new MeanAggregate(); - case MEDIAN:return new PercentileAggregate(0.50); - case PERCENTILE90:return new PercentileAggregate(0.90); - case PERCENTILE95:return new PercentileAggregate(0.95); - case PERCENTILE99:return new PercentileAggregate(0.99); - case PERCENTILE999:return new PercentileAggregate(0.999); + case PERCENTILES:return new PercentileAggregate(); } throw new IllegalStateException("unhandled enum: " + aggregate); } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java b/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java index 2794aff..a1de29d 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java @@ -1,5 +1,5 @@ package org.lucares.pdbui.domain; public enum Aggregate { - NONE, MEAN, MEDIAN, PERCENTILE90, PERCENTILE95, PERCENTILE99, PERCENTILE999 + NONE, PERCENTILES } diff --git a/pdb-ui/src/main/resources/templates/main.html b/pdb-ui/src/main/resources/templates/main.html index 7ec522f..5616f9f 100644 --- a/pdb-ui/src/main/resources/templates/main.html +++ b/pdb-ui/src/main/resources/templates/main.html @@ -17,7 +17,7 @@
+ data-autocomplete-empty-message="nothing found" value="pod=vapfinra01 and method = ViewService.findFieldViewGroup" />