From 92dde94443e20a5a1a683abf4d2512753d542fe2 Mon Sep 17 00:00:00 2001 From: ahr Date: Sun, 5 Nov 2017 09:21:34 +0100 Subject: [PATCH] preparation to add plots for percentiles --- .../pdb/plot/api/AggreateInternal.java | 2 +- .../pdb/plot/api/AggregateHandler.java | 16 ++++++ .../lucares/pdb/plot/api/MeanAggregate.java | 30 +++++++++++ .../lucares/pdb/plot/api/NullAggregate.java | 20 ++++++++ .../lucares/pdb/plot/api/PlotSettings.java | 6 +-- .../lucares/recommind/logs/CsvSummary.java | 50 +++++++++++-------- .../recommind/logs/GnuplotFileGenerator.java | 19 ++----- .../recommind/logs/GnuplotSettings.java | 7 +-- .../org/lucares/recommind/logs/Plotter.java | 16 +++--- .../pdbui/PlotSettingsTransformer.java | 12 +++-- .../org/lucares/pdbui/domain/Aggregate.java | 2 +- pdb-ui/src/main/resources/templates/main.html | 3 ++ 12 files changed, 129 insertions(+), 54 deletions(-) create mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java create mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java create mode 100644 pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggreateInternal.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggreateInternal.java index 25cdebe..c6ab8a7 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggreateInternal.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggreateInternal.java @@ -1,5 +1,5 @@ package org.lucares.pdb.plot.api; public enum AggreateInternal { -NONE, MEAN +NONE, MEAN, PERCENTILE95, PERCENTILE99, PERCENTILE999 } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java new file mode 100644 index 0000000..c35dd07 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java @@ -0,0 +1,16 @@ +package org.lucares.pdb.plot.api; + +import java.util.Collection; + +import org.lucares.recommind.logs.DataSeries; + +public interface AggregateHandler { + + void addStats(StringBuilder result, Collection dataSeries); + + default void appendfln(final StringBuilder builder, final String format, final Object... args) { + builder.append(String.format(format + "\n", args)); + } + + void addPlots(StringBuilder result, Collection dataSeries); +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java new file mode 100644 index 0000000..e04a321 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java @@ -0,0 +1,30 @@ +package org.lucares.pdb.plot.api; + +import java.util.Collection; + +import org.lucares.recommind.logs.DataSeries; + +public class MeanAggregate implements AggregateHandler{ + + + + @Override + public void addStats(StringBuilder result, Collection dataSeries) { + int count = 1; + for (final DataSeries dataSerie : dataSeries) { + + appendfln(result, "stats '%s' using 2 prefix \"A%d\"", dataSerie.getDataFile(),count); + count++; + } + } + + @Override + public void addPlots(StringBuilder result, Collection dataSeries) { + int count = 1; + for (final DataSeries dataSerie : dataSeries) { + appendfln(result, "A%d_mean title '%s Mean', \\", count, + dataSerie.getTitle(), dataSerie.getTitle()); + } + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java new file mode 100644 index 0000000..d1c948a --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java @@ -0,0 +1,20 @@ +package org.lucares.pdb.plot.api; + +import java.util.Collection; + +import org.lucares.recommind.logs.DataSeries; + +public class NullAggregate implements AggregateHandler { + + + @Override + public void addStats(StringBuilder result, Collection dataSeries) { + // nothing to do; this is a Null-Object + } + + @Override + public void addPlots(StringBuilder result, Collection dataSeries) { + // nothing to do; this is a Null-Object + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java index ca2bfc3..21ec47b 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java @@ -32,7 +32,7 @@ public class PlotSettings { private AxisScale yAxisScale; - private AggreateInternal aggregate; + private AggregateHandler aggregate; private boolean keyOutside; @@ -163,11 +163,11 @@ public class PlotSettings { + ", axisScale=" + yAxisScale + ", aggregate="+aggregate+", keyOutside="+keyOutside+"]"; } - public void setAggregate(AggreateInternal aggregate) { + public void setAggregate(AggregateHandler aggregate) { this.aggregate = aggregate; } - public AggreateInternal getAggregate() { + public AggregateHandler getAggregate() { return aggregate; } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java index 63945c7..f1432d0 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java @@ -1,21 +1,29 @@ -package org.lucares.recommind.logs; - - -class CsvSummary { - private final int values; - private long maxValue; - - public CsvSummary(final int values, long maxValue) { - super(); - this.values = values; - this.maxValue = maxValue; - } - - public int getValues() { - return values; - } - - public long getMaxValue() { - return maxValue; - } -} +package org.lucares.recommind.logs; + +import java.io.File; + + +class CsvSummary { + private final int values; + private long maxValue; + private File dataFile; + + public CsvSummary(File dataFile, final int values, long maxValue) { + super(); + this.dataFile = dataFile; + this.values = values; + this.maxValue = maxValue; + } + + public File getDataFile() { + return dataFile; + } + + public int getValues() { + return values; + } + + public long getMaxValue() { + return maxValue; + } +} diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java index 6abe50f..996b406 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -18,16 +18,8 @@ public class GnuplotFileGenerator { appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator()); - int count = 1; - if (settings.getAggregate() != AggreateInternal.NONE) - { - for (final DataSeries dataSerie : dataSeries) { - - appendfln(result, "stats '%s' using 2 prefix \"A%d\"", dataSerie.getDataFile(),count); - count++; - } - } - + settings.getAggregate().addStats(result, dataSeries); + appendfln(result, "set timefmt '%s'", settings.getTimefmt()); appendfln(result, "set xdata time"); @@ -67,16 +59,11 @@ public class GnuplotFileGenerator { appendf(result, "plot "); - count = 1; for (final DataSeries dataSerie : dataSeries) { appendfln(result, "'%s' using 1:2 title '%s' with points, \\", dataSerie.getDataFile(), dataSerie.getTitle()); - if (settings.getAggregate() == AggreateInternal.MEAN) { - appendfln(result, "A%d_mean title '%s Mean', \\", count, dataSerie.getTitle(), - dataSerie.getTitle()); - } - count++; } + settings.getAggregate().addPlots(result, dataSeries); return result.toString(); } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java index c440222..cf1ef7c 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java @@ -3,6 +3,7 @@ package org.lucares.recommind.logs; import java.nio.file.Path; import org.lucares.pdb.plot.api.AggreateInternal; +import org.lucares.pdb.plot.api.AggregateHandler; import org.lucares.pdb.plot.api.AxisScale; public class GnuplotSettings { @@ -31,7 +32,7 @@ public class GnuplotSettings { private AxisScale yAxisScale; private String dateFrom; private String dateTo; - private AggreateInternal aggregate; + private AggregateHandler aggregate; private boolean keyOutside = false; public GnuplotSettings(final Path output) { @@ -138,11 +139,11 @@ public class GnuplotSettings { return dateTo; } - public void setAggregate(AggreateInternal aggregate) { + public void setAggregate(AggregateHandler aggregate) { this.aggregate = aggregate; } - public AggreateInternal getAggregate() { + public AggregateHandler getAggregate() { return aggregate; } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java index c3bb105..91591a6 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -26,6 +26,8 @@ import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.GroupResult; import org.lucares.pdb.api.Result; import org.lucares.pdb.api.Tags; +import org.lucares.pdb.plot.api.AggreateInternal; +import org.lucares.pdb.plot.api.AggregateHandler; import org.lucares.pdb.plot.api.Limit; import org.lucares.pdb.plot.api.PlotSettings; import org.lucares.performance.db.PerformanceDb; @@ -94,11 +96,10 @@ public class Plotter { final Stream entries = groupResult.asStream(); - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - final CsvSummary csvSummary = toCsv(entries, dataFile, dateFrom, dateTo); + final CsvSummary csvSummary = toCsv(entries, tmpDir, dateFrom, dateTo, plotSettings); final String title = title(groupResult.getGroupedBy(), csvSummary.getValues()); - final DataSeries dataSerie = new DataSeries(dataFile, title, csvSummary.getValues(), csvSummary.getMaxValue()); + final DataSeries dataSerie = new DataSeries(csvSummary.getDataFile(), title, csvSummary.getValues(), csvSummary.getMaxValue()); if (dataSerie.getValues() > 0) { dataSeries.add(dataSerie); } @@ -223,14 +224,17 @@ public class Plotter { } - private static CsvSummary toCsv(final Stream entries, final File dataFile, final OffsetDateTime dateFrom, - final OffsetDateTime dateTo) throws IOException { + private static CsvSummary toCsv(final Stream entries, final Path tmpDir, final OffsetDateTime dateFrom, + final OffsetDateTime dateTo, PlotSettings plotSettings) throws IOException { + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); final long start = System.nanoTime(); int count = 0; final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); + final AggregateHandler aggregate = plotSettings.getAggregate(); + long maxValue = 0; long ignoredValues = 0; @@ -271,7 +275,7 @@ public class Plotter { } METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis)); - return new CsvSummary(count, maxValue); + return new CsvSummary(dataFile, count, maxValue); } private static String longToString(final long value){ diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 6da060c..223bedc 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -1,8 +1,11 @@ package org.lucares.pdbui; import org.lucares.pdb.plot.api.AggreateInternal; +import org.lucares.pdb.plot.api.AggregateHandler; import org.lucares.pdb.plot.api.AxisScale; import org.lucares.pdb.plot.api.Limit; +import org.lucares.pdb.plot.api.MeanAggregate; +import org.lucares.pdb.plot.api.NullAggregate; import org.lucares.pdb.plot.api.PlotSettings; import org.lucares.pdbui.domain.Aggregate; import org.lucares.pdbui.domain.LimitBy; @@ -29,10 +32,13 @@ class PlotSettingsTransformer { return result; } - private static AggreateInternal toAggregateInternal(Aggregate aggregate) { + private static AggregateHandler toAggregateInternal(Aggregate aggregate) { switch (aggregate) { - case NONE:return AggreateInternal.NONE; - case MEAN:return AggreateInternal.MEAN; + case NONE:return new NullAggregate(); + case MEAN:return new MeanAggregate(); + case PERCENTILE95:return new NullAggregate(); + case PERCENTILE99:return new NullAggregate(); + case PERCENTILE999:return new NullAggregate(); } throw new IllegalStateException("unhandled enum: " + aggregate); } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java b/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java index 3e15b08..cdd6b72 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java @@ -1,5 +1,5 @@ package org.lucares.pdbui.domain; public enum Aggregate { - NONE, MEAN + NONE, MEAN, PERCENTILE95, PERCENTILE99, PERCENTILE999 } diff --git a/pdb-ui/src/main/resources/templates/main.html b/pdb-ui/src/main/resources/templates/main.html index 50a3c9a..633fdd1 100644 --- a/pdb-ui/src/main/resources/templates/main.html +++ b/pdb-ui/src/main/resources/templates/main.html @@ -70,6 +70,9 @@