replace individual percentile aggregates with a single one for all

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java

@@ -1,5 +1,6 @@
 package org.lucares.pdb.plot.api;
 
+import java.nio.file.Path;
 import java.util.Collection;
 
 import org.lucares.recommind.logs.DataSeries;
@@ -16,6 +17,6 @@ public interface AggregateHandler {
 
 	void addPlots(StringBuilder result, Collection<DataSeries> dataSeries);
 
-	CustomAggregator createCustomAggregator(long minDate, long maxDate);
+	CustomAggregator createCustomAggregator(Path tmpDir);
 
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java

@@ -1,21 +1,21 @@
 package org.lucares.pdb.plot.api;
 
-import java.util.List;
+import java.io.File;
 
 public class AggregatedData {
-private final String label;
+	private final String label;
-private final List<AggregatedDataEntry> data;
+	private File dataFile;
-public AggregatedData(String label, List<AggregatedDataEntry> data) {
+
-	super();
+	public AggregatedData(String label, File dataFile) {
 		this.label = label;
-	this.data = data;
+		this.dataFile = dataFile;
-}
+	}
-public String getLabel() {
+
+	public String getLabel() {
 		return label;
-}
+	}
-public List<AggregatedDataEntry> getData() {
-	return data;
-}
-
 
+	public File getDataFile() {
+		return dataFile;
+	}
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java

@@ -1,9 +1,11 @@
 package org.lucares.pdb.plot.api;
 
+import java.io.IOException;
+
 public interface CustomAggregator {
 
 	void addValue(long epochMilli, long value);
 
-	AggregatedData getAggregatedData();
+	AggregatedData getAggregatedData() throws IOException;
 
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java

@@ -1,38 +0,0 @@
-package org.lucares.pdb.plot.api;
-
-import java.util.Collection;
-
-import org.lucares.recommind.logs.DataSeries;
-import org.lucares.recommind.logs.FileBackedDataSeries;
-
-public class MeanAggregate implements AggregateHandler{
-
-	@Override
-	public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
-		int count = 1;
-		for (final DataSeries dataSerie : dataSeries) {
-			if (dataSerie instanceof FileBackedDataSeries){
-				appendfln(result, "stats '%s' using 2 prefix \"A%d\"", ((FileBackedDataSeries) dataSerie).getDataFile(),count);
-				count++;
-			}
-		}
-	}
-
-	@Override
-	public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
-		int count = 1;
-		for (final DataSeries dataSerie : dataSeries) {
-			if (dataSerie instanceof FileBackedDataSeries){
-				appendfln(result, "A%d_mean title '%s Mean', \\", count,
-						dataSerie.getTitle(), dataSerie.getTitle());
-				count++;
-			}
-		}
-	}
-
-	@Override
-	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
-		return new NullCustomAggregator();
-	}
-
-}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java

@@ -1,14 +1,15 @@
 package org.lucares.pdb.plot.api;
 
+import java.nio.file.Path;
 import java.util.Collection;
 
 import org.lucares.recommind.logs.DataSeries;
 
 public class NullAggregate implements AggregateHandler {
 
-
 	@Override
-	public void addGnuplotDefinitions(StringBuilder result, String separator ,Collection<DataSeries> dataSeries) {
+	public void addGnuplotDefinitions(StringBuilder result, String separator,
+			Collection<DataSeries> dataSeries) {
 		// nothing to do; this is a Null-Object
 	}
 
@@ -18,7 +19,7 @@ public class NullAggregate implements AggregateHandler {
 	}
 
 	@Override
-	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
+	public CustomAggregator createCustomAggregator(Path tmpDir) {
 		return new NullCustomAggregator();
 	}
 

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java

@@ -1,52 +1,38 @@
 package org.lucares.pdb.plot.api;
 
+import java.nio.file.Path;
 import java.util.Collection;
 
 import org.lucares.recommind.logs.DataSeries;
 
 public class PercentileAggregate implements AggregateHandler{
 
-	private double percentile;
+	@Override
+	public CustomAggregator createCustomAggregator(Path tmpDir) {
+		return new PercentileCustomAggregator(tmpDir);
+	}
 	
-	/**
+	
-	 * 
+	public PercentileAggregate() {
-	 * @param percentile range > 0.0 and < 1.0)
-	 */
-	public PercentileAggregate(double percentile) {
-		this.percentile = percentile;
 	}
 
 	@Override
 	public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
 		
-		for (DataSeries dataSerie : dataSeries) {
+	// TODO still neeeded?
-			if (dataSerie.getAggregatedData() != null){
-				
-				appendfln(result, "$%s << EOD", dataSerie.getId());
-				
-				for (AggregatedDataEntry aggregatedData : dataSerie.getAggregatedData().getData()) {
-					appendfln(result, "%.3f%s%d", aggregatedData.getEpochSeconds(), separator, aggregatedData.getValue());
-				}
-				
-				
-				appendfln(result, "EOD", dataSerie.getId());
-			}
-		}
 		
 	}
 
 	@Override
 	public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
 		for (DataSeries dataSerie : dataSeries) {			
-			if (dataSerie.getAggregatedData() != null){
+			final AggregatedData aggregatedData = dataSerie.getAggregatedData();
-				appendfln(result, "'$%s' using 1:2 title '%s' with linespoints lw 2, \\", dataSerie.getId(), dataSerie.getTitle()+" " +dataSerie.getAggregatedData().getLabel());
+			if (aggregatedData != null){
+				appendfln(result, "'%s' using 1:2 title '%s' with lines lw 1 axes x2y1, \\", //
+						aggregatedData.getDataFile().getAbsolutePath(),// 
+						dataSerie.getTitle()+" " +aggregatedData.getLabel());
 			}
 		}
 	}
 
-	@Override
-	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
-		return new PercentileCustomAggregator(percentile, minDate, maxDate);
-	}
-
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java

@@ -1,54 +1,68 @@
 package org.lucares.pdb.plot.api;
 
-import java.util.ArrayList;
+import java.io.BufferedWriter;
-import java.util.List;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 
 import org.lucares.collections.IntList;
 
 public class PercentileCustomAggregator implements CustomAggregator{
 
-	private double percentile;
+	private final static int POINTS = 300;
-	private long fromEpochMilli;
-	private long numBuckets;
-	private long bucketSize;
 	
-	private List<IntList> buckets = new ArrayList<>();
+	private final IntList values = new IntList(); // TODO should be a LongList
 
-	public PercentileCustomAggregator(double percentile, long fromEpochMilli, long toEpochMilli) {
+	private Path tmpDir;
-		this.percentile = percentile;
+
-		this.fromEpochMilli = fromEpochMilli;
+	
-		this.numBuckets = 50;
+
-		this.bucketSize = (toEpochMilli - fromEpochMilli) / numBuckets;
+	public PercentileCustomAggregator(Path tmpDir) {
-		for(int i = 0; i < numBuckets; i++){
+		this.tmpDir = tmpDir;
-			buckets.add(new IntList());
-		}
 	}
 
 	@Override
 	public void addValue(long epochMilli, long value) {
-		final int bucket = (int) ((epochMilli  - fromEpochMilli ) / bucketSize);
+		values.add((int) value);
-		buckets.get(bucket).add((int)value); // TODO we should use a LongList instead of an IntList
 	}
 
 	@Override
-	public AggregatedData getAggregatedData() {
+	public AggregatedData getAggregatedData() throws IOException {
-		final List<AggregatedDataEntry> data = new ArrayList<>((int)numBuckets);
+		final char separator = ',';
+		final char newline = '\n';
 		
-		for (int i= 0; i < numBuckets; i++) {
+		values.parallelSort();
-			final IntList values = buckets.get(i);
-			values.sort();
-			final int indexOfPercentile = (int) (values.size() * percentile);
-			final long value = values.size() > indexOfPercentile ? values.get(indexOfPercentile) : 0;
-			final long epochMilli = fromEpochMilli + i*bucketSize + bucketSize/2; // the middle of the bucket
-			final double epochSecond = epochMilli / 1000.0; // gnuplot wants the time in seconds
-			final AggregatedDataEntry aggregatedDataEntry = new AggregatedDataEntry(epochSecond, value);
 		
-			if (value > 0){
+		
-				data.add(aggregatedDataEntry );
+		final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
+		try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
+			
+			final StringBuilder data = new StringBuilder();
+			if (values.size() > 0) {
+				// compute the percentiles
+				for (int i = 0; i < POINTS; i++) {
+					data.append(i* (100/(double)POINTS));
+					data.append(separator);
+					data.append(values.get((int) Math.floor(values.size()
+							/ ((double)POINTS) * i)));
+					data.append(newline);
 				}
+				final int maxValue = values.get(values.size() - 1);
+				data.append(100);
+				data.append(separator);
+				data.append(maxValue);
+				data.append(newline);
 			}
-		final String title = String.format("%.1f%% percentile", percentile*100);
+			output.write(data.toString());
-		return new AggregatedData(title, data);
+		}
+		
+		
+		final String title = String.format("percentiles");
+		return new AggregatedData(title, dataFile);
 	}
 
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -30,6 +30,18 @@ public class GnuplotFileGenerator {
 		appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel());
 		appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo());
 		
+		/*
+		 * set yrange ["0":"100"]
+			set ytics nomirror # don't show the tics of the left y-axis on the right side
+			
+			set y2label "CPU load"
+			set y2tics
+			set autoscale y2
+		 */
+		appendfln(result, "set x2label \"percent\"");
+		appendfln(result, "set y2tics");
+		appendfln(result, "set x2range [\"0\":\"100\"]");
+		
 		final long graphOffset = settings.getYAxisScale() == AxisScale.LINEAR ? 0 : 1;
 		appendfln(result, "set yrange [\""+graphOffset+"\":]");
 		

pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java

@@ -171,7 +171,7 @@ public class ScatterPlot implements ConcretePlotter {
 		final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
 		final long toEpochMilli = dateTo.toInstant().toEpochMilli();
 		final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
-		final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
+		final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir);
 		
 		long maxValue = 0;
 		long ignoredValues = 0;

pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java

@@ -1,7 +1,6 @@
 package org.lucares.pdbui;
 
 import org.lucares.pdb.plot.api.AggregateHandler;
-import org.lucares.pdb.plot.api.MeanAggregate;
 import org.lucares.pdb.plot.api.NullAggregate;
 import org.lucares.pdb.plot.api.PercentileAggregate;
 import org.lucares.pdb.plot.api.PlotSettings;
@@ -32,12 +31,7 @@ class PlotSettingsTransformer {
 	private static AggregateHandler toAggregateInternal(Aggregate aggregate) {
 		switch (aggregate) {
 		case NONE:return new NullAggregate();
-		case MEAN:return new MeanAggregate();
+		case PERCENTILES:return new PercentileAggregate();
-		case MEDIAN:return new PercentileAggregate(0.50);
-		case PERCENTILE90:return new PercentileAggregate(0.90);
-		case PERCENTILE95:return new PercentileAggregate(0.95);
-		case PERCENTILE99:return new PercentileAggregate(0.99);
-		case PERCENTILE999:return new PercentileAggregate(0.999);
 		}
 		throw new IllegalStateException("unhandled enum: " + aggregate);
 	}

pdb-ui/src/main/java/org/lucares/pdbui/domain/Aggregate.java

@@ -1,5 +1,5 @@
 package org.lucares.pdbui.domain;
 
 public enum Aggregate {
-	NONE, MEAN, MEDIAN, PERCENTILE90, PERCENTILE95, PERCENTILE99, PERCENTILE999
+	NONE, PERCENTILES
 }

pdb-ui/src/main/resources/templates/main.html

@@ -17,7 +17,7 @@
 	
 	<div id="search-input-wrapper">
 		<input id="search-input" placeholder="field=value and anotherField=anotherValue" data-autocomplete="autocomplete"
-			data-autocomplete-empty-message="nothing found" />
+			data-autocomplete-empty-message="nothing found" value="pod=vapfinra01 and method = ViewService.findFieldViewGroup" />
 	</div>
 	<div id="search-bar">
 		<form>
@@ -78,12 +78,7 @@
 					<label for="show-aggregate">Aggregate:</label>
 					<select id="show-aggregate">
 						<option value="NONE" selected="selected">-</option>
-						<option value="MEAN">Mean</option>
+						<option value="PERCENTILES">percentiles</option>
-						<option value="MEDIAN">Median</option>
-						<option value="PERCENTILE90">90% percentile</option>
-						<option value="PERCENTILE95">95% percentile</option>
-						<option value="PERCENTILE99">99% percentile</option>
-						<option value="PERCENTILE999">99.9% percentile</option>
 					</select>
 				</div>
 				<div class="group">