andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

improvements

Browse Source 
- split the 'sortby' select field into two fields
- sort by average
- legend shows plotted and total values in the date range
- removed InlineDataSeries, because it was not used anymore
This commit is contained in:
Andreas Huber
2018-05-01 17:32:25 +02:00
parent 82dca3a885
commit bda2de672e
11 changed files with 284 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java
View File

@@ -4,13 +4,11 @@ import java.io.File;
public class AggregatedData {
private final String label;
private File dataFile;
private double average;
private final File dataFile;
public AggregatedData(String label, File dataFile, double average) {
public AggregatedData(final String label, final File dataFile) {
this.label = label;
this.dataFile = dataFile;
this.average = average;
}
public String getLabel() {
@@ -20,8 +18,4 @@ public class AggregatedData {
public File getDataFile() {
return dataFile;
}
public double getAverage() {
return average;
}
}

4
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java
View File

@@ -61,11 +61,9 @@ public class PercentileCustomAggregator implements CustomAggregator {
output.write(data.toString());
}
// TODO remove:
final double average = percentiles.stream().summaryStatistics().getAverage();
final String title = String.format("percentiles");
return new AggregatedData(title, dataFile, average);
return new AggregatedData(title, dataFile);
}
}

34
pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java
View File

@@ -4,18 +4,22 @@ import java.io.File;
import org.lucares.pdb.plot.api.AggregatedData;
class CsvSummary {
private final int values;
private long maxValue;
private File dataFile;
private AggregatedData aggregatedData;
private final long maxValue;
private final File dataFile;
private final AggregatedData aggregatedData;
private final double statsAverage;
private final int plottedValues;
public CsvSummary(File dataFile, final int values, long maxValue, AggregatedData aggregatedData) {
public CsvSummary(final File dataFile, final int values, final int plottedValues, final long maxValue,
final double statsAverage, final AggregatedData aggregatedData) {
super();
this.dataFile = dataFile;
this.values = values;
this.plottedValues = plottedValues;
this.maxValue = maxValue;
this.statsAverage = statsAverage;
this.aggregatedData = aggregatedData;
}
@@ -23,14 +27,34 @@ class CsvSummary {
return dataFile;
}
/**
* Total number of values in the selected date range.
*
* @see CsvSummary#getPlottedValues()
* @return total number of values
*/
public int getValues() {
return values;
}
/**
* Number of plotted values in the selected date range <em>and</em> y-range.
*
* @see CsvSummary#getValues()
* @return number of plotted values
*/
public int getPlottedValues() {
return plottedValues;
}
public long getMaxValue() {
return maxValue;
}
public double getStatsAverage() {
return statsAverage;
}
public AggregatedData getAggregatedData() {
return aggregatedData;
}

4
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -30,8 +30,12 @@ public interface DataSeries {
public int getValues();
public int getPlottedValues();
public long getMaxValue();
public double getAverage();
public void setStyle(String style);
public String getStyle();

37
pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java
View File

@@ -8,22 +8,23 @@ public class FileBackedDataSeries implements DataSeries {
private final String title;
private CsvSummary csvSummary;
private final CsvSummary csvSummary;
private int id;
private final int id;
private GnuplotLineType linetype;
private final GnuplotLineType linetype;
private String style;
public FileBackedDataSeries(int id, String title, CsvSummary csvSummary,
GnuplotLineType linetype) {
public FileBackedDataSeries(final int id, final String title, final CsvSummary csvSummary,
final GnuplotLineType linetype) {
this.id = id;
this.title = title;
this.csvSummary = csvSummary;
this.linetype = linetype;
}
@Override
public String getIdAsString() {
return "id" + id;
}
@@ -34,7 +35,7 @@ public class FileBackedDataSeries implements DataSeries {
}
@Override
public void setStyle(String style) {
public void setStyle(final String style) {
this.style = style;
}
@@ -47,34 +48,44 @@ public class FileBackedDataSeries implements DataSeries {
return csvSummary.getDataFile();
}
@Override
public String getTitle() {
return title;
}
@Override
public int getValues() {
return csvSummary.getValues();
}
@Override
public int getPlottedValues() {
return csvSummary.getPlottedValues();
}
@Override
public long getMaxValue() {
return csvSummary.getMaxValue();
}
@Override
public double getAverage() {
return csvSummary.getStatsAverage();
}
@Override
public AggregatedData getAggregatedData() {
return csvSummary.getAggregatedData();
}
@Override
public String getGnuplotPlotDefinition() {
String average="";
if (getAggregatedData() != null){
average = String.format(" [%.2f]", getAggregatedData().getAverage());
}
return String.format("'%s' using 1:2 title '%s' with %s %s, \\", //
getDataFile(),//
getTitle()+average,//
getDataFile(), //
getTitle(), //
linetype, // line or points
style//
);
);
}
}

71
pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java
View File

@@ -1,71 +0,0 @@
package org.lucares.recommind.logs;
import org.lucares.pdb.plot.api.AggregatedData;
public class InlineDataSeries implements DataSeries{
private long maxValue;
private int numValues;
private String title;
private int id;
private String inlineData;
private String style;
public InlineDataSeries(long maxValue, int numValues, String title,
int id, String inlineData) {
this.maxValue = maxValue;
this.numValues = numValues;
this.title = title;
this.id = id;
this.inlineData = inlineData;
}
@Override
public String getIdAsString() {
return "id"+id;
}
@Override
public int getId() {
return id;
}
@Override
public void setStyle(String style) {
this.style = style;
}
@Override
public String getStyle() {
return style;
}
@Override
public String getTitle() {
return title;
}
@Override
public int getValues() {
return numValues;
}
@Override
public long getMaxValue() {
return maxValue;
}
@Override
public AggregatedData getAggregatedData() {
return null;
}
@Override
public String getGnuplotPlotDefinition() {
return String.format("'-' u 1:2 title '%s' with line\n%s,", title, inlineData);
}
}

34
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java
View File

@@ -90,7 +90,7 @@ public class ScatterPlot {
final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
final int id = idCounter.incrementAndGet();
final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
final String title = title(groupResult.getGroupedBy(), csvSummary);
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary,
GnuplotLineType.Points);
if (dataSerie.getValues() > 0) {
@@ -201,7 +201,6 @@ public class ScatterPlot {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
final long start = System.nanoTime();
final Stream<Entry> entries = groupResult.asStream();
int count = 0;
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
@@ -213,7 +212,10 @@ public class ScatterPlot {
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir);
int count = 0; // number of values in the x-axis range (used to compute stats)
int plottedValues = 0;
long statsMaxValue = 0;
double statsCurrentAverage = 0.0;
long ignoredValues = 0;
final int separator = ',';
final int newline = '\n';
@@ -234,6 +236,15 @@ public class ScatterPlot {
final long value = entry.getValue();
aggregator.addValue(epochMilli, value);
// compute stats
count++;
statsMaxValue = Math.max(statsMaxValue, value);
// compute average (important to do this after 'count' has been incremented)
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
// check if value is in the selected y-range
if (value < minValue || value > maxValue) {
ignoredValues++;
continue;
@@ -255,16 +266,16 @@ public class ScatterPlot {
output.write(stringValue);
output.write(newline);
count++;
statsMaxValue = Math.max(statsMaxValue, value);
plottedValues++;
}
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
groupResult.getGroupedBy().asString(), dataFile);
return new CsvSummary(dataFile, count, statsMaxValue, aggregator.getAggregatedData());
return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
aggregator.getAggregatedData());
}
static String uniqueDirectoryName() {
@@ -272,10 +283,13 @@ public class ScatterPlot {
+ UUID.randomUUID().toString();
}
static String title(final Tags tags, final int values) {
static String title(final Tags tags, final CsvSummary csvSummary) {
final StringBuilder result = new StringBuilder();
final int values = csvSummary.getValues();
final int plottedValues = csvSummary.getPlottedValues();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
@@ -288,7 +302,11 @@ public class ScatterPlot {
}
result.append(" (");
result.append(String.format("%,d", values));
if (plottedValues != values) {
result.append(String.format("%,d/%,d", plottedValues, values));
} else {
result.append(String.format("%,d", values));
}
result.append(")");
return result.toString();

64
pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java Normal file
View File

@@ -0,0 +1,64 @@
package org.lucares.pdbui.domain;
import java.util.Collection;
public class DataSeriesStats {
private final int values;
private final long maxValue;
private final double average;
private final int plottedValues;
public DataSeriesStats(final int values, final int plottedValues, final long maxValue, final double average) {
this.values = values;
this.plottedValues = plottedValues;
this.maxValue = maxValue;
this.average = average;
}
/**
* The number of values in the date range, without applying the y-range.
*
* @return total number of values
*/
public int getValues() {
return values;
}
/**
* The number of values in the date range <em>and</em> the y-range.
*
* @return number of plotted values
*/
public int getPlottedValues() {
return plottedValues;
}
public long getMaxValue() {
return maxValue;
}
public double getAverage() {
return average;
}
@Override
public String toString() {
return "[values=" + values + ", maxValue=" + maxValue + ", average=" + average + "]";
}
public static double average(final Collection<DataSeriesStats> stats) {
long n = 0;
double average = 0;
for (final DataSeriesStats stat : stats) {
final int newValues = stat.getValues();
final double newAverage = stat.getAverage();
if (newValues > 0) {
average = (average * n + newAverage * newValues) / (n + newValues);
n += newValues;
}
}
return average;
}
}

50
pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java
View File

@@ -1,5 +1,6 @@
package org.lucares.pdbui.domain;
import java.util.ArrayList;
import java.util.List;
import org.lucares.recommind.logs.DataSeries;
@@ -9,14 +10,24 @@ public class PlotResponseStats {
private int values;
private double average;
private int plottedValues;
private List<DataSeriesStats> dataSeriesStats;
public PlotResponseStats() {
super();
}
public PlotResponseStats(final long maxValue, final int values) {
public PlotResponseStats(final long maxValue, final int values, final int plottedValues, final double average,
final List<DataSeriesStats> dataSeriesStats) {
this.maxValue = maxValue;
this.values = values;
this.plottedValues = plottedValues;
this.average = average;
this.dataSeriesStats = dataSeriesStats;
}
public long getMaxValue() {
@@ -35,21 +46,54 @@ public class PlotResponseStats {
this.values = values;
}
public int getPlottedValues() {
return plottedValues;
}
public void setPlottedValues(final int plottedValues) {
this.plottedValues = plottedValues;
}
public double getAverage() {
return average;
}
public void setAverage(final double average) {
this.average = average;
}
public List<DataSeriesStats> getDataSeriesStats() {
return dataSeriesStats;
}
public void setDataSeriesStats(final List<DataSeriesStats> dataSeriesStats) {
this.dataSeriesStats = dataSeriesStats;
}
@Override
public String toString() {
return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + "]";
return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + ", average=" + average
+ ", plottedValues=" + plottedValues + ", dataSeriesStats=" + dataSeriesStats + "]";
}
public static PlotResponseStats fromDataSeries(final List<DataSeries> dataSeries) {
int values = 0;
int plottedValues = 0;
long maxValue = 0;
final List<DataSeriesStats> dataSeriesStats = new ArrayList<>();
for (final DataSeries dataSerie : dataSeries) {
values += dataSerie.getValues();
plottedValues += dataSerie.getPlottedValues();
maxValue = Math.max(maxValue, dataSerie.getMaxValue());
dataSeriesStats.add(new DataSeriesStats(dataSerie.getValues(), dataSerie.getPlottedValues(),
dataSerie.getMaxValue(), dataSerie.getAverage()));
}
return new PlotResponseStats(maxValue, values);
final double average = DataSeriesStats.average(dataSeriesStats);
return new PlotResponseStats(maxValue, values, plottedValues, average, dataSeriesStats);
}
}

37
pdb-ui/src/main/resources/resources/js/ui.js
View File

@@ -308,10 +308,14 @@ Vue.component('navigation-bar-gallery', {
<label for="gallerySortBy">Sort by:</label>
<select id="gallerySortBy" name="gallerySortBy" v-model="gallery.sortBy" @change="sort">
<option value="DEFAULT"></option>
<option value="MAX_VALUE_DESC">max value desc</option>
<option value="MAX_VALUE_ASC">max value asc</option>
<option value="VALUES_DESC">#values desc</option>
<option value="VALUES_ASC">#values asc</option>
<option value="MAX_VALUE">max value</option>
<option value="AVERAGE">average</option>
<option value="VALUES">#values</option>
<option value="PLOTTED_VALUES">#plotted values</option>
</select>
<select id="gallerySortOrder" name="gallerySortOrder" v-model="gallery.sortOrder" @change="sort">
<option value="DESC">desc</option>
<option value="ASC">asc</option>
</select>
</div>
<!--
@@ -603,6 +607,8 @@ Vue.component('search-bar', {
'yRangeMax': data.searchBar.yRange.max,
'yRangeUnit': data.searchBar.yRange.unit,
'keyOutside': data.searchBar.keyOutside,
'sortyBy': data.gallery.sortBy,
'sortyOrder': data.gallery.sortOrder,
};
var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params );
@@ -833,7 +839,8 @@ var data = {
tiles: [],
toBeRendered: [],
image: "",
sortBy: "MAX_VALUE_DESC",
sortBy: GetURLParameter('sortyBy','MAX_VALUE'),
sortOrder: GetURLParameter('sortyOrder','DESC'),
filter: "NONE",
progress: {
max: 0,
@@ -1063,21 +1070,23 @@ function createGalleryItem(originalQuery, field, imageHeight, imageWidth)
function sortTiles() {
const orderFactor = data.gallery.sortOrder == 'ASC' ? 1 : -1;
switch (data.gallery.sortBy) {
case "DEFAULT":
data.gallery.tiles.sort(function(a, b){return a.fieldValue.localeCompare(b.fieldValue);});
data.gallery.tiles.sort(function(a, b){return orderFactor*a.fieldValue.localeCompare(b.fieldValue);});
break;
case "VALUES_ASC":
data.gallery.tiles.sort(function(a, b){return a.stats.values - b.stats.values;});
case "VALUES":
data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.values - b.stats.values);});
break;
case "VALUES_DESC":
data.gallery.tiles.sort(function(a, b){return b.stats.values - a.stats.values;});
case "PLOTTED_VALUES":
data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.plottedValues - b.stats.plottedValues);});
break;
case "MAX_VALUE_ASC":
data.gallery.tiles.sort(function(a, b){return a.stats.maxValue - b.stats.maxValue;});
case "MAX_VALUE":
data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.maxValue - b.stats.maxValue);});
break;
case "MAX_VALUE_DESC":
data.gallery.tiles.sort(function(a, b){return b.stats.maxValue - a.stats.maxValue;});
case "AVERAGE":
data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.average - b.stats.average);});
break;
}
}

60
pdb-ui/src/test/java/org/lucares/pdbui/domain/DataSeriesStatsTest.java Normal file
View File

@@ -0,0 +1,60 @@
package org.lucares.pdbui.domain;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
public class DataSeriesStatsTest {
@DataProvider
public Object[][] providerAverage() {
final List<Object[]> result = new ArrayList<>();
{
final List<DataSeriesStats> stats = Arrays.asList(//
new DataSeriesStats(10, 0, 0, 5.0)//
);
final double expected = 5.0;
result.add(new Object[] { stats, expected });
}
{
final List<DataSeriesStats> stats = Arrays.asList(//
new DataSeriesStats(0, 0, 0, 5.0)//
);
final double expected = 0.0; // no values
result.add(new Object[] { stats, expected });
}
{
final List<DataSeriesStats> stats = Arrays.asList(//
new DataSeriesStats(10, 0, 0, 5.0), //
new DataSeriesStats(40, 0, 0, 1.0)//
);
final double expected = 1.8; // 90 / 50
result.add(new Object[] { stats, expected });
}
{
final List<DataSeriesStats> stats = Arrays.asList(//
new DataSeriesStats(5, 0, 0, 7.0), //
new DataSeriesStats(0, 0, 0, 5.0), // // no values
new DataSeriesStats(20, 0, 0, 2.0)//
);
final double expected = 3.0; // (35+40) / 25
result.add(new Object[] { stats, expected });
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "providerAverage")
public void testAverage(final Collection<DataSeriesStats> stats, final double expected) {
final double actual = DataSeriesStats.average(stats);
Assert.assertEquals(actual, expected, 0.01);
}
}