From bda2de672e9097bd747ab4c52edaaf6cabdc457a Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 1 May 2018 17:32:25 +0200 Subject: [PATCH] improvements - split the 'sortby' select field into two fields - sort by average - legend shows plotted and total values in the date range - removed InlineDataSeries, because it was not used anymore --- .../lucares/pdb/plot/api/AggregatedData.java | 10 +-- .../plot/api/PercentileCustomAggregator.java | 4 +- .../lucares/recommind/logs/CsvSummary.java | 40 ++++++++--- .../lucares/recommind/logs/DataSeries.java | 4 ++ .../recommind/logs/FileBackedDataSeries.java | 39 ++++++---- .../recommind/logs/InlineDataSeries.java | 71 ------------------- .../lucares/recommind/logs/ScatterPlot.java | 34 ++++++--- .../lucares/pdbui/domain/DataSeriesStats.java | 64 +++++++++++++++++ .../pdbui/domain/PlotResponseStats.java | 50 ++++++++++++- pdb-ui/src/main/resources/resources/js/ui.js | 37 ++++++---- .../pdbui/domain/DataSeriesStatsTest.java | 60 ++++++++++++++++ 11 files changed, 284 insertions(+), 129 deletions(-) delete mode 100644 pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java create mode 100644 pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java create mode 100644 pdb-ui/src/test/java/org/lucares/pdbui/domain/DataSeriesStatsTest.java diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java index c796f2f..37ec819 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java @@ -4,13 +4,11 @@ import java.io.File; public class AggregatedData { private final String label; - private File dataFile; - private double average; + private final File dataFile; - public AggregatedData(String label, File dataFile, double average) { + public AggregatedData(final String label, final File dataFile) { this.label = label; this.dataFile = dataFile; - this.average = average; } public String getLabel() { @@ -20,8 +18,4 @@ public class AggregatedData { public File getDataFile() { return dataFile; } - - public double getAverage() { - return average; - } } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java index 04c22ed..308be19 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java @@ -61,11 +61,9 @@ public class PercentileCustomAggregator implements CustomAggregator { output.write(data.toString()); } - // TODO remove: - final double average = percentiles.stream().summaryStatistics().getAverage(); final String title = String.format("percentiles"); - return new AggregatedData(title, dataFile, average); + return new AggregatedData(title, dataFile); } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java index 1f39cce..aab42dd 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java @@ -4,33 +4,57 @@ import java.io.File; import org.lucares.pdb.plot.api.AggregatedData; - class CsvSummary { private final int values; - private long maxValue; - private File dataFile; - private AggregatedData aggregatedData; + private final long maxValue; + private final File dataFile; + private final AggregatedData aggregatedData; + private final double statsAverage; + private final int plottedValues; - public CsvSummary(File dataFile, final int values, long maxValue, AggregatedData aggregatedData) { + public CsvSummary(final File dataFile, final int values, final int plottedValues, final long maxValue, + final double statsAverage, final AggregatedData aggregatedData) { super(); this.dataFile = dataFile; this.values = values; + this.plottedValues = plottedValues; this.maxValue = maxValue; + this.statsAverage = statsAverage; this.aggregatedData = aggregatedData; } - + public File getDataFile() { return dataFile; } + /** + * Total number of values in the selected date range. + * + * @see CsvSummary#getPlottedValues() + * @return total number of values + */ public int getValues() { return values; } - + + /** + * Number of plotted values in the selected date range and y-range. + * + * @see CsvSummary#getValues() + * @return number of plotted values + */ + public int getPlottedValues() { + return plottedValues; + } + public long getMaxValue() { return maxValue; } - + + public double getStatsAverage() { + return statsAverage; + } + public AggregatedData getAggregatedData() { return aggregatedData; } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java index c1fdc1d..95b82e6 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java @@ -30,8 +30,12 @@ public interface DataSeries { public int getValues(); + public int getPlottedValues(); + public long getMaxValue(); + public double getAverage(); + public void setStyle(String style); public String getStyle(); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java index 61fd239..a2a35fe 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java @@ -8,22 +8,23 @@ public class FileBackedDataSeries implements DataSeries { private final String title; - private CsvSummary csvSummary; + private final CsvSummary csvSummary; - private int id; + private final int id; - private GnuplotLineType linetype; + private final GnuplotLineType linetype; private String style; - public FileBackedDataSeries(int id, String title, CsvSummary csvSummary, - GnuplotLineType linetype) { + public FileBackedDataSeries(final int id, final String title, final CsvSummary csvSummary, + final GnuplotLineType linetype) { this.id = id; this.title = title; this.csvSummary = csvSummary; this.linetype = linetype; } + @Override public String getIdAsString() { return "id" + id; } @@ -34,7 +35,7 @@ public class FileBackedDataSeries implements DataSeries { } @Override - public void setStyle(String style) { + public void setStyle(final String style) { this.style = style; } @@ -47,34 +48,44 @@ public class FileBackedDataSeries implements DataSeries { return csvSummary.getDataFile(); } + @Override public String getTitle() { return title; } + @Override public int getValues() { return csvSummary.getValues(); } + @Override + public int getPlottedValues() { + return csvSummary.getPlottedValues(); + } + + @Override public long getMaxValue() { return csvSummary.getMaxValue(); } + @Override + public double getAverage() { + return csvSummary.getStatsAverage(); + } + + @Override public AggregatedData getAggregatedData() { return csvSummary.getAggregatedData(); } @Override public String getGnuplotPlotDefinition() { - String average=""; - if (getAggregatedData() != null){ - average = String.format(" [%.2f]", getAggregatedData().getAverage()); - } - + return String.format("'%s' using 1:2 title '%s' with %s %s, \\", // - getDataFile(),// - getTitle()+average,// + getDataFile(), // + getTitle(), // linetype, // line or points style// - ); + ); } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java deleted file mode 100644 index 2d12d73..0000000 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.lucares.recommind.logs; - -import org.lucares.pdb.plot.api.AggregatedData; - -public class InlineDataSeries implements DataSeries{ - - private long maxValue; - private int numValues; - private String title; - private int id; - private String inlineData; - private String style; - - public InlineDataSeries(long maxValue, int numValues, String title, - int id, String inlineData) { - this.maxValue = maxValue; - this.numValues = numValues; - this.title = title; - this.id = id; - this.inlineData = inlineData; - } - - @Override - public String getIdAsString() { - - return "id"+id; - } - - @Override - public int getId() { - return id; - } - - @Override - public void setStyle(String style) { - this.style = style; - } - - @Override - public String getStyle() { - return style; - } - - @Override - public String getTitle() { - return title; - } - - @Override - public int getValues() { - - return numValues; - } - - @Override - public long getMaxValue() { - return maxValue; - } - - @Override - public AggregatedData getAggregatedData() { - return null; - } - - @Override - public String getGnuplotPlotDefinition() { - - return String.format("'-' u 1:2 title '%s' with line\n%s,", title, inlineData); - } - -} diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java index 9ee0a25..41a6e0e 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java @@ -90,7 +90,7 @@ public class ScatterPlot { final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings); final int id = idCounter.incrementAndGet(); - final String title = title(groupResult.getGroupedBy(), csvSummary.getValues()); + final String title = title(groupResult.getGroupedBy(), csvSummary); final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.Points); if (dataSerie.getValues() > 0) { @@ -201,7 +201,6 @@ public class ScatterPlot { final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); final long start = System.nanoTime(); final Stream entries = groupResult.asStream(); - int count = 0; final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); @@ -213,7 +212,10 @@ public class ScatterPlot { final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir); + int count = 0; // number of values in the x-axis range (used to compute stats) + int plottedValues = 0; long statsMaxValue = 0; + double statsCurrentAverage = 0.0; long ignoredValues = 0; final int separator = ','; final int newline = '\n'; @@ -234,6 +236,15 @@ public class ScatterPlot { final long value = entry.getValue(); aggregator.addValue(epochMilli, value); + + // compute stats + count++; + statsMaxValue = Math.max(statsMaxValue, value); + + // compute average (important to do this after 'count' has been incremented) + statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count; + + // check if value is in the selected y-range if (value < minValue || value > maxValue) { ignoredValues++; continue; @@ -255,16 +266,16 @@ public class ScatterPlot { output.write(stringValue); output.write(newline); - count++; - statsMaxValue = Math.max(statsMaxValue, value); - + plottedValues++; } } METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy().asString(), dataFile); - return new CsvSummary(dataFile, count, statsMaxValue, aggregator.getAggregatedData()); + return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage, + aggregator.getAggregatedData()); + } static String uniqueDirectoryName() { @@ -272,10 +283,13 @@ public class ScatterPlot { + UUID.randomUUID().toString(); } - static String title(final Tags tags, final int values) { + static String title(final Tags tags, final CsvSummary csvSummary) { final StringBuilder result = new StringBuilder(); + final int values = csvSummary.getValues(); + final int plottedValues = csvSummary.getPlottedValues(); + if (tags.isEmpty()) { result.append(DEFAULT_GROUP); } else { @@ -288,7 +302,11 @@ public class ScatterPlot { } result.append(" ("); - result.append(String.format("%,d", values)); + if (plottedValues != values) { + result.append(String.format("%,d/%,d", plottedValues, values)); + } else { + result.append(String.format("%,d", values)); + } result.append(")"); return result.toString(); diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java b/pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java new file mode 100644 index 0000000..335ec64 --- /dev/null +++ b/pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java @@ -0,0 +1,64 @@ +package org.lucares.pdbui.domain; + +import java.util.Collection; + +public class DataSeriesStats { + private final int values; + private final long maxValue; + private final double average; + private final int plottedValues; + + public DataSeriesStats(final int values, final int plottedValues, final long maxValue, final double average) { + this.values = values; + this.plottedValues = plottedValues; + this.maxValue = maxValue; + this.average = average; + } + + /** + * The number of values in the date range, without applying the y-range. + * + * @return total number of values + */ + public int getValues() { + return values; + } + + /** + * The number of values in the date range and the y-range. + * + * @return number of plotted values + */ + public int getPlottedValues() { + return plottedValues; + } + + public long getMaxValue() { + return maxValue; + } + + public double getAverage() { + return average; + } + + @Override + public String toString() { + return "[values=" + values + ", maxValue=" + maxValue + ", average=" + average + "]"; + } + + public static double average(final Collection stats) { + long n = 0; + double average = 0; + + for (final DataSeriesStats stat : stats) { + final int newValues = stat.getValues(); + final double newAverage = stat.getAverage(); + if (newValues > 0) { + average = (average * n + newAverage * newValues) / (n + newValues); + n += newValues; + } + } + + return average; + } +} diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java b/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java index 29e7df0..bd42262 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java @@ -1,5 +1,6 @@ package org.lucares.pdbui.domain; +import java.util.ArrayList; import java.util.List; import org.lucares.recommind.logs.DataSeries; @@ -9,14 +10,24 @@ public class PlotResponseStats { private int values; + private double average; + + private int plottedValues; + + private List dataSeriesStats; + public PlotResponseStats() { super(); } - public PlotResponseStats(final long maxValue, final int values) { + public PlotResponseStats(final long maxValue, final int values, final int plottedValues, final double average, + final List dataSeriesStats) { this.maxValue = maxValue; this.values = values; + this.plottedValues = plottedValues; + this.average = average; + this.dataSeriesStats = dataSeriesStats; } public long getMaxValue() { @@ -35,21 +46,54 @@ public class PlotResponseStats { this.values = values; } + public int getPlottedValues() { + return plottedValues; + } + + public void setPlottedValues(final int plottedValues) { + this.plottedValues = plottedValues; + } + + public double getAverage() { + return average; + } + + public void setAverage(final double average) { + this.average = average; + } + + public List getDataSeriesStats() { + return dataSeriesStats; + } + + public void setDataSeriesStats(final List dataSeriesStats) { + this.dataSeriesStats = dataSeriesStats; + } + @Override public String toString() { - return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + "]"; + return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + ", average=" + average + + ", plottedValues=" + plottedValues + ", dataSeriesStats=" + dataSeriesStats + "]"; } public static PlotResponseStats fromDataSeries(final List dataSeries) { int values = 0; + int plottedValues = 0; long maxValue = 0; + final List dataSeriesStats = new ArrayList<>(); for (final DataSeries dataSerie : dataSeries) { values += dataSerie.getValues(); + plottedValues += dataSerie.getPlottedValues(); maxValue = Math.max(maxValue, dataSerie.getMaxValue()); + + dataSeriesStats.add(new DataSeriesStats(dataSerie.getValues(), dataSerie.getPlottedValues(), + dataSerie.getMaxValue(), dataSerie.getAverage())); } - return new PlotResponseStats(maxValue, values); + final double average = DataSeriesStats.average(dataSeriesStats); + + return new PlotResponseStats(maxValue, values, plottedValues, average, dataSeriesStats); } } diff --git a/pdb-ui/src/main/resources/resources/js/ui.js b/pdb-ui/src/main/resources/resources/js/ui.js index da598e7..63d23a4 100644 --- a/pdb-ui/src/main/resources/resources/js/ui.js +++ b/pdb-ui/src/main/resources/resources/js/ui.js @@ -308,10 +308,14 @@ Vue.component('navigation-bar-gallery', { +