improvements

- split the 'sortby' select field into two fields
- sort by average
- legend shows plotted and total values in the date range
- removed InlineDataSeries, because it was not used anymore

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java

@@ -4,13 +4,11 @@ import java.io.File;
 
 public class AggregatedData {
 	private final String label;
-	private File dataFile;
+	private final File dataFile;
-	private double average;
 
-	public AggregatedData(String label, File dataFile, double average) {
+	public AggregatedData(final String label, final File dataFile) {
 		this.label = label;
 		this.dataFile = dataFile;
-		this.average = average;
 	}
 
 	public String getLabel() {
@@ -20,8 +18,4 @@ public class AggregatedData {
 	public File getDataFile() {
 		return dataFile;
 	}
-	
-	public double getAverage() {
-		return average;
-	}
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java

@@ -61,11 +61,9 @@ public class PercentileCustomAggregator implements CustomAggregator {
 			output.write(data.toString());
 
 		}
-		// TODO remove:
-		final double average = percentiles.stream().summaryStatistics().getAverage();
 
 		final String title = String.format("percentiles");
-		return new AggregatedData(title, dataFile, average);
+		return new AggregatedData(title, dataFile);
 	}
 
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java

@@ -4,18 +4,22 @@ import java.io.File;
 
 import org.lucares.pdb.plot.api.AggregatedData;
 
-
 class CsvSummary {
 	private final int values;
-	private long maxValue;
+	private final long maxValue;
-	private File dataFile;
+	private final File dataFile;
-	private AggregatedData aggregatedData;
+	private final AggregatedData aggregatedData;
+	private final double statsAverage;
+	private final int plottedValues;
 
-	public CsvSummary(File dataFile, final int values,  long maxValue, AggregatedData aggregatedData) {
+	public CsvSummary(final File dataFile, final int values, final int plottedValues, final long maxValue,
+			final double statsAverage, final AggregatedData aggregatedData) {
 		super();
 		this.dataFile = dataFile;
 		this.values = values;
+		this.plottedValues = plottedValues;
 		this.maxValue = maxValue;
+		this.statsAverage = statsAverage;
 		this.aggregatedData = aggregatedData;
 	}
 
@@ -23,14 +27,34 @@ class CsvSummary {
 		return dataFile;
 	}
 
+	/**
+	 * Total number of values in the selected date range.
+	 *
+	 * @see CsvSummary#getPlottedValues()
+	 * @return total number of values
+	 */
 	public int getValues() {
 		return values;
 	}
 
+	/**
+	 * Number of plotted values in the selected date range <em>and</em> y-range.
+	 *
+	 * @see CsvSummary#getValues()
+	 * @return number of plotted values
+	 */
+	public int getPlottedValues() {
+		return plottedValues;
+	}
+
 	public long getMaxValue() {
 		return maxValue;
 	}
 
+	public double getStatsAverage() {
+		return statsAverage;
+	}
+
 	public AggregatedData getAggregatedData() {
 		return aggregatedData;
 	}

pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java

@@ -30,8 +30,12 @@ public interface DataSeries {
 
 	public int getValues();
 
+	public int getPlottedValues();
+
 	public long getMaxValue();
 
+	public double getAverage();
+
 	public void setStyle(String style);
 
 	public String getStyle();

pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java

@@ -8,22 +8,23 @@ public class FileBackedDataSeries implements DataSeries {
 
 	private final String title;
 
-	private CsvSummary csvSummary;
+	private final CsvSummary csvSummary;
 
-	private int id;
+	private final int id;
 
-	private GnuplotLineType linetype;
+	private final GnuplotLineType linetype;
 
 	private String style;
 
-	public FileBackedDataSeries(int id, String title, CsvSummary csvSummary,
+	public FileBackedDataSeries(final int id, final String title, final CsvSummary csvSummary,
-			GnuplotLineType linetype) {
+			final GnuplotLineType linetype) {
 		this.id = id;
 		this.title = title;
 		this.csvSummary = csvSummary;
 		this.linetype = linetype;
 	}
 
+	@Override
 	public String getIdAsString() {
 		return "id" + id;
 	}
@@ -34,7 +35,7 @@ public class FileBackedDataSeries implements DataSeries {
 	}
 
 	@Override
-	public void setStyle(String style) {
+	public void setStyle(final String style) {
 		this.style = style;
 	}
 
@@ -47,34 +48,44 @@ public class FileBackedDataSeries implements DataSeries {
 		return csvSummary.getDataFile();
 	}
 
+	@Override
 	public String getTitle() {
 		return title;
 	}
 
+	@Override
 	public int getValues() {
 		return csvSummary.getValues();
 	}
 
+	@Override
+	public int getPlottedValues() {
+		return csvSummary.getPlottedValues();
+	}
+
+	@Override
 	public long getMaxValue() {
 		return csvSummary.getMaxValue();
 	}
 
+	@Override
+	public double getAverage() {
+		return csvSummary.getStatsAverage();
+	}
+
+	@Override
 	public AggregatedData getAggregatedData() {
 		return csvSummary.getAggregatedData();
 	}
 
 	@Override
 	public String getGnuplotPlotDefinition() {
-		String average="";
-		if (getAggregatedData() != null){
-			average = String.format(" [%.2f]", getAggregatedData().getAverage());
-		}
 
 		return String.format("'%s' using 1:2 title '%s' with %s %s, \\", //
-				getDataFile(),//
+				getDataFile(), //
-				getTitle()+average,//
+				getTitle(), //
 				linetype, // line or points
 				style//
-				);
+		);
 	}
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/InlineDataSeries.java

@@ -1,71 +0,0 @@
-package org.lucares.recommind.logs;
-
-import org.lucares.pdb.plot.api.AggregatedData;
-
-public class InlineDataSeries implements DataSeries{
-
-	private long maxValue;
-	private int numValues;
-	private String title;
-	private int id;
-	private String inlineData;
-	private String style;
-
-	public InlineDataSeries(long maxValue, int numValues, String title,
-			int id, String inlineData) {
-		this.maxValue = maxValue;
-		this.numValues = numValues;
-		this.title = title;
-		this.id = id;
-		this.inlineData = inlineData;
-	}
-
-	@Override
-	public String getIdAsString() {
-
-		return "id"+id;
-	}
-	
-	@Override
-	public int getId() {
-		return id;
-	}
-	
-	@Override
-	public void setStyle(String style) {
-		this.style = style;
-	}
-
-	@Override
-	public String getStyle() {
-		return style;
-	}
-
-	@Override
-	public String getTitle() {
-		return title;
-	}
-
-	@Override
-	public int getValues() {
-		
-		return numValues;
-	}
-
-	@Override
-	public long getMaxValue() {
-		return maxValue;
-	}
-
-	@Override
-	public AggregatedData getAggregatedData() {
-		return null;
-	}
-
-	@Override
-	public String getGnuplotPlotDefinition() {
-		
-		return String.format("'-' u 1:2 title '%s' with line\n%s,", title, inlineData);
-	}
-
-}

pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java

@@ -90,7 +90,7 @@ public class ScatterPlot {
 					final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
 
 					final int id = idCounter.incrementAndGet();
-					final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
+					final String title = title(groupResult.getGroupedBy(), csvSummary);
 					final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary,
 							GnuplotLineType.Points);
 					if (dataSerie.getValues() > 0) {
@@ -201,7 +201,6 @@ public class ScatterPlot {
 		final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
 		final long start = System.nanoTime();
 		final Stream<Entry> entries = groupResult.asStream();
-		int count = 0;
 		final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
 		final long toEpochMilli = dateTo.toInstant().toEpochMilli();
 		final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
@@ -213,7 +212,10 @@ public class ScatterPlot {
 
 		final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir);
 
+		int count = 0; // number of values in the x-axis range (used to compute stats)
+		int plottedValues = 0;
 		long statsMaxValue = 0;
+		double statsCurrentAverage = 0.0;
 		long ignoredValues = 0;
 		final int separator = ',';
 		final int newline = '\n';
@@ -234,6 +236,15 @@ public class ScatterPlot {
 
 				final long value = entry.getValue();
 				aggregator.addValue(epochMilli, value);
+
+				// compute stats
+				count++;
+				statsMaxValue = Math.max(statsMaxValue, value);
+
+				// compute average (important to do this after 'count' has been incremented)
+				statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
+
+				// check if value is in the selected y-range
 				if (value < minValue || value > maxValue) {
 					ignoredValues++;
 					continue;
@@ -255,16 +266,16 @@ public class ScatterPlot {
 				output.write(stringValue);
 				output.write(newline);
 
-				count++;
+				plottedValues++;
-				statsMaxValue = Math.max(statsMaxValue, value);
-
 			}
 		}
 
 		METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
 				count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
 				groupResult.getGroupedBy().asString(), dataFile);
-		return new CsvSummary(dataFile, count, statsMaxValue, aggregator.getAggregatedData());
+		return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
+				aggregator.getAggregatedData());
+
 	}
 
 	static String uniqueDirectoryName() {
@@ -272,10 +283,13 @@ public class ScatterPlot {
 				+ UUID.randomUUID().toString();
 	}
 
-	static String title(final Tags tags, final int values) {
+	static String title(final Tags tags, final CsvSummary csvSummary) {
 
 		final StringBuilder result = new StringBuilder();
 
+		final int values = csvSummary.getValues();
+		final int plottedValues = csvSummary.getPlottedValues();
+
 		if (tags.isEmpty()) {
 			result.append(DEFAULT_GROUP);
 		} else {
@@ -288,7 +302,11 @@ public class ScatterPlot {
 		}
 
 		result.append(" (");
-		result.append(String.format("%,d", values));
+		if (plottedValues != values) {
+			result.append(String.format("%,d/%,d", plottedValues, values));
+		} else {
+			result.append(String.format("%,d", values));
+		}
 		result.append(")");
 
 		return result.toString();

pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java

@@ -0,0 +1,64 @@
+package org.lucares.pdbui.domain;
+
+import java.util.Collection;
+
+public class DataSeriesStats {
+	private final int values;
+	private final long maxValue;
+	private final double average;
+	private final int plottedValues;
+
+	public DataSeriesStats(final int values, final int plottedValues, final long maxValue, final double average) {
+		this.values = values;
+		this.plottedValues = plottedValues;
+		this.maxValue = maxValue;
+		this.average = average;
+	}
+
+	/**
+	 * The number of values in the date range, without applying the y-range.
+	 *
+	 * @return total number of values
+	 */
+	public int getValues() {
+		return values;
+	}
+
+	/**
+	 * The number of values in the date range <em>and</em> the y-range.
+	 *
+	 * @return number of plotted values
+	 */
+	public int getPlottedValues() {
+		return plottedValues;
+	}
+
+	public long getMaxValue() {
+		return maxValue;
+	}
+
+	public double getAverage() {
+		return average;
+	}
+
+	@Override
+	public String toString() {
+		return "[values=" + values + ", maxValue=" + maxValue + ", average=" + average + "]";
+	}
+
+	public static double average(final Collection<DataSeriesStats> stats) {
+		long n = 0;
+		double average = 0;
+
+		for (final DataSeriesStats stat : stats) {
+			final int newValues = stat.getValues();
+			final double newAverage = stat.getAverage();
+			if (newValues > 0) {
+				average = (average * n + newAverage * newValues) / (n + newValues);
+				n += newValues;
+			}
+		}
+
+		return average;
+	}
+}

pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java

@@ -1,5 +1,6 @@
 package org.lucares.pdbui.domain;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.lucares.recommind.logs.DataSeries;
@@ -9,14 +10,24 @@ public class PlotResponseStats {
 
 	private int values;
 
+	private double average;
+
+	private int plottedValues;
+
+	private List<DataSeriesStats> dataSeriesStats;
+
 	public PlotResponseStats() {
 		super();
 	}
 
-	public PlotResponseStats(final long maxValue, final int values) {
+	public PlotResponseStats(final long maxValue, final int values, final int plottedValues, final double average,
+			final List<DataSeriesStats> dataSeriesStats) {
 
 		this.maxValue = maxValue;
 		this.values = values;
+		this.plottedValues = plottedValues;
+		this.average = average;
+		this.dataSeriesStats = dataSeriesStats;
 	}
 
 	public long getMaxValue() {
@@ -35,21 +46,54 @@ public class PlotResponseStats {
 		this.values = values;
 	}
 
+	public int getPlottedValues() {
+		return plottedValues;
+	}
+
+	public void setPlottedValues(final int plottedValues) {
+		this.plottedValues = plottedValues;
+	}
+
+	public double getAverage() {
+		return average;
+	}
+
+	public void setAverage(final double average) {
+		this.average = average;
+	}
+
+	public List<DataSeriesStats> getDataSeriesStats() {
+		return dataSeriesStats;
+	}
+
+	public void setDataSeriesStats(final List<DataSeriesStats> dataSeriesStats) {
+		this.dataSeriesStats = dataSeriesStats;
+	}
+
 	@Override
 	public String toString() {
-		return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + "]";
+		return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + ", average=" + average
+				+ ", plottedValues=" + plottedValues + ", dataSeriesStats=" + dataSeriesStats + "]";
 	}
 
 	public static PlotResponseStats fromDataSeries(final List<DataSeries> dataSeries) {
 
 		int values = 0;
+		int plottedValues = 0;
 		long maxValue = 0;
+		final List<DataSeriesStats> dataSeriesStats = new ArrayList<>();
 
 		for (final DataSeries dataSerie : dataSeries) {
 			values += dataSerie.getValues();
+			plottedValues += dataSerie.getPlottedValues();
 			maxValue = Math.max(maxValue, dataSerie.getMaxValue());
+
+			dataSeriesStats.add(new DataSeriesStats(dataSerie.getValues(), dataSerie.getPlottedValues(),
+					dataSerie.getMaxValue(), dataSerie.getAverage()));
 		}
 
-		return new PlotResponseStats(maxValue, values);
+		final double average = DataSeriesStats.average(dataSeriesStats);
+
+		return new PlotResponseStats(maxValue, values, plottedValues, average, dataSeriesStats);
 	}
 }

pdb-ui/src/main/resources/resources/js/ui.js

@@ -308,10 +308,14 @@ Vue.component('navigation-bar-gallery', {
 			<label for="gallerySortBy">Sort by:</label>
 			<select id="gallerySortBy" name="gallerySortBy" v-model="gallery.sortBy" @change="sort">
 				<option value="DEFAULT"></option>
-				<option value="MAX_VALUE_DESC">max value desc</option>
+				<option value="MAX_VALUE">max value</option>
-				<option value="MAX_VALUE_ASC">max value asc</option>
+				<option value="AVERAGE">average</option>
-				<option value="VALUES_DESC">#values desc</option>
+				<option value="VALUES">#values</option>
-				<option value="VALUES_ASC">#values asc</option>
+				<option value="PLOTTED_VALUES">#plotted values</option>
+			</select>
+			<select id="gallerySortOrder" name="gallerySortOrder" v-model="gallery.sortOrder" @change="sort">
+				<option value="DESC">desc</option>
+				<option value="ASC">asc</option>
 			</select>
 		</div>
 		<!--
@@ -603,6 +607,8 @@ Vue.component('search-bar', {
 	    	'yRangeMax': data.searchBar.yRange.max,
 	    	'yRangeUnit': data.searchBar.yRange.unit,
 	    	'keyOutside': data.searchBar.keyOutside,
+	    	'sortyBy': data.gallery.sortBy,
+	    	'sortyOrder': data.gallery.sortOrder,
 		};
 	
 		var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params );
@@ -833,7 +839,8 @@ var data = {
 		tiles: [],
 		toBeRendered: [],
 		image: "",
-		sortBy: "MAX_VALUE_DESC",
+		sortBy: GetURLParameter('sortyBy','MAX_VALUE'),
+		sortOrder: GetURLParameter('sortyOrder','DESC'),
 		filter: "NONE",
 		progress: {
 			max: 0,
@@ -1063,21 +1070,23 @@ function createGalleryItem(originalQuery, field, imageHeight, imageWidth)
 
 function sortTiles() {
 
+	const orderFactor = data.gallery.sortOrder == 'ASC' ? 1 : -1;
+
 	switch (data.gallery.sortBy) {
 		case "DEFAULT":
-			data.gallery.tiles.sort(function(a, b){return a.fieldValue.localeCompare(b.fieldValue);});
+			data.gallery.tiles.sort(function(a, b){return orderFactor*a.fieldValue.localeCompare(b.fieldValue);});
 			break;
-		case "VALUES_ASC":
+		case "VALUES":
-			data.gallery.tiles.sort(function(a, b){return a.stats.values - b.stats.values;});
+			data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.values - b.stats.values);});
 			break;
-		case "VALUES_DESC":
+		case "PLOTTED_VALUES":
-			data.gallery.tiles.sort(function(a, b){return b.stats.values - a.stats.values;});
+			data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.plottedValues - b.stats.plottedValues);});
 			break;
-		case "MAX_VALUE_ASC":
+		case "MAX_VALUE":
-			data.gallery.tiles.sort(function(a, b){return a.stats.maxValue - b.stats.maxValue;});
+			data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.maxValue - b.stats.maxValue);});
 			break;
-		case "MAX_VALUE_DESC":
+		case "AVERAGE":
-			data.gallery.tiles.sort(function(a, b){return b.stats.maxValue - a.stats.maxValue;});
+			data.gallery.tiles.sort(function(a, b){return orderFactor*(a.stats.average - b.stats.average);});
 			break;
 	}
 }

pdb-ui/src/test/java/org/lucares/pdbui/domain/DataSeriesStatsTest.java

@@ -0,0 +1,60 @@
+package org.lucares.pdbui.domain;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+@Test
+public class DataSeriesStatsTest {
+	@DataProvider
+	public Object[][] providerAverage() {
+		final List<Object[]> result = new ArrayList<>();
+
+		{
+			final List<DataSeriesStats> stats = Arrays.asList(//
+					new DataSeriesStats(10, 0, 0, 5.0)//
+			);
+			final double expected = 5.0;
+			result.add(new Object[] { stats, expected });
+		}
+		{
+			final List<DataSeriesStats> stats = Arrays.asList(//
+					new DataSeriesStats(0, 0, 0, 5.0)//
+			);
+			final double expected = 0.0; // no values
+			result.add(new Object[] { stats, expected });
+		}
+
+		{
+			final List<DataSeriesStats> stats = Arrays.asList(//
+					new DataSeriesStats(10, 0, 0, 5.0), //
+					new DataSeriesStats(40, 0, 0, 1.0)//
+			);
+			final double expected = 1.8; // 90 / 50
+			result.add(new Object[] { stats, expected });
+		}
+		{
+			final List<DataSeriesStats> stats = Arrays.asList(//
+					new DataSeriesStats(5, 0, 0, 7.0), //
+					new DataSeriesStats(0, 0, 0, 5.0), // // no values
+					new DataSeriesStats(20, 0, 0, 2.0)//
+			);
+			final double expected = 3.0; // (35+40) / 25
+			result.add(new Object[] { stats, expected });
+		}
+
+		return result.toArray(new Object[0][]);
+	}
+
+	@Test(dataProvider = "providerAverage")
+	public void testAverage(final Collection<DataSeriesStats> stats, final double expected) {
+
+		final double actual = DataSeriesStats.average(stats);
+		Assert.assertEquals(actual, expected, 0.01);
+	}
+}