add plots for percentiles

pdb-api/src/test/java/org/lucares/memory/MemoryScale.java

@@ -25,6 +25,7 @@ public class MemoryScale {
 		runGc();
 		long memoryAfter = getUsedMemory();
 		System.out.println("used memory: " + (memoryAfter - memoryBefore));
+		handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler
 	}
 	
 	private static Object createObject(){

pdb-plotting/build.gradle

@@ -1,6 +1,7 @@
 
 dependencies {
 	compile project(':performanceDb')
+	compile 'org.lucares:primitiveCollections:0.1.20171007100354'
 	compile 'com.fasterxml.jackson.core:jackson-databind:2.9.1'
 	compile 'com.google.guava:guava:23.0'
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java

@@ -6,11 +6,14 @@ import org.lucares.recommind.logs.DataSeries;
 
 public interface AggregateHandler {
 	
-	void addStats(StringBuilder result, Collection<DataSeries> dataSeries);
+	void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries);
 	
 	default void appendfln(final StringBuilder builder, final String format, final Object... args) {
 		builder.append(String.format(format + "\n", args));
 	}
 
 	void addPlots(StringBuilder result, Collection<DataSeries> dataSeries);
+
+	CustomAggregator createCustomAggregator(long minDate, long maxDate);
+
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java

@@ -0,0 +1,21 @@
+package org.lucares.pdb.plot.api;
+
+import java.util.List;
+
+public class AggregatedData {
+private final String label;
+private final List<AggregatedDataEntry> data;
+public AggregatedData(String label, List<AggregatedDataEntry> data) {
+	super();
+	this.label = label;
+	this.data = data;
+}
+public String getLabel() {
+	return label;
+}
+public List<AggregatedDataEntry> getData() {
+	return data;
+}
+
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataEntry.java

@@ -0,0 +1,18 @@
+package org.lucares.pdb.plot.api;
+
+public class AggregatedDataEntry {
+private final double epochSeconds;
+private final long value;
+public AggregatedDataEntry(double epochSeconds, long value) {
+	super();
+	this.epochSeconds = epochSeconds;
+	this.value = value;
+}
+public double getEpochSeconds() {
+	return epochSeconds;
+}
+public long getValue() {
+	return value;
+}
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java

@@ -0,0 +1,9 @@
+package org.lucares.pdb.plot.api;
+
+public interface CustomAggregator {
+
+	void addValue(long epochMilli, long value);
+
+	AggregatedData getAggregatedData();
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java

@@ -9,7 +9,7 @@ public class MeanAggregate implements AggregateHandler{
 	
 
 	@Override
-	public void addStats(StringBuilder result, Collection<DataSeries> dataSeries) {
+	public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
 		int count = 1;
 		for (final DataSeries dataSerie : dataSeries) {
 			
@@ -24,7 +24,13 @@ public class MeanAggregate implements AggregateHandler{
 		for (final DataSeries dataSerie : dataSeries) {
 			appendfln(result, "A%d_mean title '%s Mean', \\", count,
 					dataSerie.getTitle(), dataSerie.getTitle());
+			count++;
 		}
 	}
 
+	@Override
+	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
+		return new NullCustomAggregator();
+	}
+
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java

@@ -8,7 +8,7 @@ public class NullAggregate implements AggregateHandler {
 
 
 	@Override
-	public void addStats(StringBuilder result, Collection<DataSeries> dataSeries) {
+	public void addGnuplotDefinitions(StringBuilder result, String separator ,Collection<DataSeries> dataSeries) {
 		// nothing to do; this is a Null-Object
 	}
 
@@ -17,4 +17,9 @@ public class NullAggregate implements AggregateHandler {
 		// nothing to do; this is a Null-Object
 	}
 
+	@Override
+	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
+		return new NullCustomAggregator();
+	}
+
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullCustomAggregator.java

@@ -0,0 +1,16 @@
+package org.lucares.pdb.plot.api;
+
+public class NullCustomAggregator implements CustomAggregator{
+
+	@Override
+	public void addValue(long epochMilli, long value) {
+		// TODO Auto-generated method stub
+		
+	}
+
+	@Override
+	public AggregatedData getAggregatedData() {
+		return null;
+	}
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java

@@ -0,0 +1,50 @@
+package org.lucares.pdb.plot.api;
+
+import java.util.Collection;
+
+import org.lucares.recommind.logs.DataSeries;
+
+public class PercentileAggregate implements AggregateHandler{
+
+	private double percentile;
+
+	/**
+	 * 
+	 * @param percentile range > 0.0 and < 1.0)
+	 */
+	public PercentileAggregate(double percentile) {
+		this.percentile = percentile;
+	}
+
+	@Override
+	public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
+		
+		for (DataSeries dataSerie : dataSeries) {
+			if (dataSerie.getAggregatedData() != null){
+				
+				appendfln(result, "$%s << EOD", dataSerie.getId());
+				
+				for (AggregatedDataEntry aggregatedData : dataSerie.getAggregatedData().getData()) {
+					appendfln(result, "%.3f%s%d", aggregatedData.getEpochSeconds(), separator, aggregatedData.getValue());
+				}
+				
+				
+				appendfln(result, "EOD", dataSerie.getId());
+			}
+		}
+		
+	}
+
+	@Override
+	public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
+		for (DataSeries dataSerie : dataSeries) {			
+			appendfln(result, "'$%s' using 1:2 title '%s' with line, \\", dataSerie.getId(), dataSerie.getTitle()+" " +dataSerie.getAggregatedData().getLabel());
+		}
+	}
+
+	@Override
+	public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
+		return new PercentileCustomAggregator(percentile, minDate, maxDate);
+	}
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java

@@ -0,0 +1,54 @@
+package org.lucares.pdb.plot.api;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.lucares.collections.IntList;
+
+public class PercentileCustomAggregator implements CustomAggregator{
+
+	private double percentile;
+	private long fromEpochMilli;
+	private long numBuckets;
+	private long bucketSize;
+	
+	private List<IntList> buckets = new ArrayList<>();
+
+	public PercentileCustomAggregator(double percentile, long fromEpochMilli, long toEpochMilli) {
+		this.percentile = percentile;
+		this.fromEpochMilli = fromEpochMilli;
+		this.numBuckets = 50;
+		this.bucketSize = (toEpochMilli - fromEpochMilli) / numBuckets;
+		for(int i = 0; i < numBuckets; i++){
+			buckets.add(new IntList());
+		}
+	}
+
+	@Override
+	public void addValue(long epochMilli, long value) {
+		final int bucket = (int) ((epochMilli  - fromEpochMilli ) / bucketSize);
+		buckets.get(bucket).add((int)value); // TODO we should use a LongList instead of an IntList
+	}
+
+	@Override
+	public AggregatedData getAggregatedData() {
+		final List<AggregatedDataEntry> data = new ArrayList<>((int)numBuckets);
+		
+		for (int i= 0; i < numBuckets; i++) {
+			final IntList values = buckets.get(i);
+			values.sort();
+			final int indexOfPercentile = (int) (values.size() * percentile);
+			final long value = values.size() > indexOfPercentile ? values.get(indexOfPercentile) : 0;
+			final long epochMilli = fromEpochMilli + i*bucketSize + bucketSize/2; // the middle of the bucket
+			final double epochSecond = epochMilli / 1000.0; // gnuplot wants the time in seconds
+			final AggregatedDataEntry aggregatedDataEntry = new AggregatedDataEntry(epochSecond, value);
+			
+			if (value > 0){
+				data.add(aggregatedDataEntry );
+			}
+		}
+		final String title = String.format("%.1f%% percentile", percentile*100);
+		return new AggregatedData(title, data);
+	}
+
+}

pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java

@@ -2,17 +2,21 @@ package org.lucares.recommind.logs;
 
 import java.io.File;
 
+import org.lucares.pdb.plot.api.AggregatedData;
+
 
 class CsvSummary {
 	private final int values;
 	private long maxValue;
 	private File dataFile;
+	private AggregatedData aggregatedData;
 
-	public CsvSummary(File dataFile, final int values,  long maxValue) {
+	public CsvSummary(File dataFile, final int values,  long maxValue, AggregatedData aggregatedData) {
 		super();
 		this.dataFile = dataFile;
 		this.values = values;
 		this.maxValue = maxValue;
+		this.aggregatedData = aggregatedData;
 	}
 	
 	public File getDataFile() {
@@ -26,4 +30,8 @@ class CsvSummary {
 	public long getMaxValue() {
 		return maxValue;
 	}
+	
+	public AggregatedData getAggregatedData() {
+		return aggregatedData;
+	}
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java

@@ -1,76 +1,70 @@
-package org.lucares.recommind.logs;
+package org.lucares.recommind.logs;
-
+
-import java.io.File;
+import java.io.File;
-import java.util.Comparator;
+import java.util.Comparator;
-import java.util.LinkedHashMap;
+import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.List;
-import java.util.Map;
+import java.util.Map;
-
+
-public class DataSeries {
+import org.lucares.pdb.plot.api.AggregatedData;
-	public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (a, b) -> {
+
-		return a.getValues() - b.getValues();
+public class DataSeries {
-	};
+	public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (
-	
+			a, b) -> {
-	public static final Comparator<? super DataSeries> BY_MAX_VALUE = (a, b) -> {
+		return a.getValues() - b.getValues();
-		final long result = a.getMaxValue() - b.getMaxValue();
+	};
-		return result <0 ? -1 : (result > 0 ? 1 : 0);
+
-	};
+	public static final Comparator<? super DataSeries> BY_MAX_VALUE = (a, b) -> {
-
+		final long result = a.getMaxValue() - b.getMaxValue();
-	private final File dataFile;
+		return result < 0 ? -1 : (result > 0 ? 1 : 0);
-
+	};
-	private final String title;
+
-
+	private final String title;
-	private final GnuplotColor color;
+
-
+	private CsvSummary csvSummary;
-	private final Integer pointType;
+
-
+	private String id;
-	private final int values;
+
-
+	public DataSeries(String id, String title, CsvSummary csvSummary) {
-	private long maxValue;
+		this.id = id;
-
+		this.title = title;
-	public DataSeries(final File dataFile, final String title, final int values, long maxValue) {
+		this.csvSummary = csvSummary;
-		super();
+	}
-		this.dataFile = dataFile;
+	public String getId() {
-		this.title = title;
+		return id;
-		this.values = values;
+	}
-		this.maxValue = maxValue;
+	
-		this.color = null;
+	public File getDataFile() {
-		this.pointType = null;
+		return csvSummary.getDataFile();
-	}
+	}
-
+
-	public GnuplotColor getColor() {
+	public String getTitle() {
-		return color;
+		return title;
-	}
+	}
-
+
-	public Integer getPointType() {
+	public int getValues() {
-		return pointType;
+		return csvSummary.getValues();
-	}
+	}
-
+
-	public File getDataFile() {
+	public long getMaxValue() {
-		return dataFile;
+		return csvSummary.getMaxValue();
-	}
+	}
-
+	
-	public String getTitle() {
+	public AggregatedData getAggregatedData() {
-		return title;
+		return csvSummary.getAggregatedData();
-	}
+	}
-
+
-	public int getValues() {
+	public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
-		return values;
+		final Map<String, Integer> result = new LinkedHashMap<>();
-	}
+
-	
+		for (final DataSeries dataSerie : dataSeries) {
-	public long getMaxValue() {
+
-		return maxValue;
+			result.put(dataSerie.getTitle(), dataSerie.getValues());
-	}
+
-
+		}
-	public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
+
-		final Map<String, Integer> result = new LinkedHashMap<>();
+		return result;
-
+	}
-		for (final DataSeries dataSerie : dataSeries) {
+
-
+	
-			result.put(dataSerie.getTitle(), dataSerie.values);
+}
-
-		}
-
-		return result;
-	}
-}

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -2,7 +2,6 @@ package org.lucares.recommind.logs;
 
 import java.util.Collection;
 
-import org.lucares.pdb.plot.api.AggreateInternal;
 import org.lucares.pdb.plot.api.AxisScale;
 
 public class GnuplotFileGenerator {
@@ -18,7 +17,7 @@ public class GnuplotFileGenerator {
 		
 		appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator());
 		
-		settings.getAggregate().addStats(result, dataSeries);
+		settings.getAggregate().addGnuplotDefinitions(result, settings.getDatafileSeparator(), dataSeries);
 	
 		appendfln(result, "set timefmt '%s'", settings.getTimefmt());
 

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java

@@ -2,7 +2,6 @@ package org.lucares.recommind.logs;
 
 import java.nio.file.Path;
 
-import org.lucares.pdb.plot.api.AggreateInternal;
 import org.lucares.pdb.plot.api.AggregateHandler;
 import org.lucares.pdb.plot.api.AxisScale;
 

pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java

@@ -26,8 +26,7 @@ import org.lucares.pdb.api.Entry;
 import org.lucares.pdb.api.GroupResult;
 import org.lucares.pdb.api.Result;
 import org.lucares.pdb.api.Tags;
-import org.lucares.pdb.plot.api.AggreateInternal;
+import org.lucares.pdb.plot.api.CustomAggregator;
-import org.lucares.pdb.plot.api.AggregateHandler;
 import org.lucares.pdb.plot.api.Limit;
 import org.lucares.pdb.plot.api.PlotSettings;
 import org.lucares.performance.db.PerformanceDb;
@@ -92,6 +91,7 @@ public class Plotter {
 
 			final Result result = db.get(query, groupBy);
 
+			int idCounter = 0;
 			for (final GroupResult groupResult : result.getGroups()) {
 
 				final Stream<Entry> entries = groupResult.asStream();
@@ -99,10 +99,11 @@ public class Plotter {
 				final CsvSummary csvSummary = toCsv(entries, tmpDir, dateFrom, dateTo, plotSettings);
 
 				final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
-				final DataSeries dataSerie = new DataSeries(csvSummary.getDataFile(), title, csvSummary.getValues(), csvSummary.getMaxValue());
+				final DataSeries dataSerie = new DataSeries("id"+idCounter, title, csvSummary);
 				if (dataSerie.getValues() > 0) {
 					dataSeries.add(dataSerie);
 				}
+				idCounter++;
 			}
 
 			if (dataSeries.isEmpty()) {
@@ -233,8 +234,7 @@ public class Plotter {
 		final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
 		final long toEpochMilli = dateTo.toInstant().toEpochMilli();
 		final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
-		final AggregateHandler aggregate = plotSettings.getAggregate();
+		final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
-		
 		
 		long maxValue = 0;
 		long ignoredValues = 0;
@@ -248,26 +248,30 @@ public class Plotter {
 			while (it.hasNext()) {
 				final Entry entry = it.next();
 
-				if (fromEpochMilli <= entry.getEpochMilli() && entry.getEpochMilli() <= toEpochMilli) {
+				long epochMilli = entry.getEpochMilli();
+				if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
 					
-					final String value = longToString(entry.getValue());
+					long value = entry.getValue();
+					final String stringValue = longToString(value);
 					final String formattedDate;
 					
 					if (useMillis){
 						formattedDateBuilder.delete(0, formattedDateBuilder.length());
-						formatter.format("%.3f", entry.getEpochMilli() / 1000.0);
+						formatter.format("%.3f", epochMilli / 1000.0);
 						formattedDate = formattedDateBuilder.toString();
 					}else {
-						formattedDate = String.valueOf(entry.getEpochMilli() / 1000);
+						formattedDate = String.valueOf(epochMilli / 1000);
 					}
 					
 					output.write(formattedDate);
 					output.write(separator);
-					output.write(value);
+					output.write(stringValue);
 					output.write(newline);
 
+					aggregator.addValue(epochMilli, value);
+					
 					count++;
-					maxValue = Math.max(maxValue, entry.getValue());
+					maxValue = Math.max(maxValue, value);
 				}else {
 					ignoredValues++;
 				}
@@ -275,7 +279,7 @@ public class Plotter {
 		}
 
 		METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis));
-		return new CsvSummary(dataFile, count,  maxValue);
+		return new CsvSummary(dataFile, count,  maxValue, aggregator.getAggregatedData());
 	}
 	
 	private static String longToString(final long value){

pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java

@@ -1,11 +1,11 @@
 package org.lucares.pdbui;
 
-import org.lucares.pdb.plot.api.AggreateInternal;
 import org.lucares.pdb.plot.api.AggregateHandler;
 import org.lucares.pdb.plot.api.AxisScale;
 import org.lucares.pdb.plot.api.Limit;
 import org.lucares.pdb.plot.api.MeanAggregate;
 import org.lucares.pdb.plot.api.NullAggregate;
+import org.lucares.pdb.plot.api.PercentileAggregate;
 import org.lucares.pdb.plot.api.PlotSettings;
 import org.lucares.pdbui.domain.Aggregate;
 import org.lucares.pdbui.domain.LimitBy;
@@ -36,9 +36,9 @@ class PlotSettingsTransformer {
 		switch (aggregate) {
 		case NONE:return new NullAggregate();
 		case MEAN:return new MeanAggregate();
-		case PERCENTILE95:return new NullAggregate();
+		case PERCENTILE95:return new PercentileAggregate(0.95);
-		case PERCENTILE99:return new NullAggregate();
+		case PERCENTILE99:return new PercentileAggregate(0.99);
-		case PERCENTILE999:return new NullAggregate();
+		case PERCENTILE999:return new PercentileAggregate(0.999);
 		}
 		throw new IllegalStateException("unhandled enum: " + aggregate);
 	}

pdb-utils/src/test/java/org/lucares/utils/MiniMapTest.java

@@ -1,7 +1,5 @@
 package org.lucares.utils;
 
-import java.util.Map;
-
 import org.testng.Assert;
 import org.testng.annotations.Test;
 

performanceDb/build.gradle

@@ -1,14 +1,14 @@
-
+
-dependencies {
+dependencies {
-	compile project(':pdb-api')
+	compile project(':pdb-api')
-	compile project(':data-store')
+	compile project(':data-store')
-	compile project(':file-utils')
+	compile project(':file-utils')
-	//compile 'org.lucares:ludb:1.0.20170408081113'
+	//compile 'org.lucares:ludb:0.1.20171007100354'
-	compile 'com.fasterxml.jackson.core:jackson-databind:2.9.1'
+	compile 'com.fasterxml.jackson.core:jackson-databind:2.9.1'
-	compile 'org.apache.commons:commons-collections4:4.1'
+	compile 'org.apache.commons:commons-collections4:4.1'
-	
+	
-	
+	
-	compile 'org.apache.logging.log4j:log4j-api:2.9.1'
+	compile 'org.apache.logging.log4j:log4j-api:2.9.1'
-	compile 'org.apache.logging.log4j:log4j-core:2.9.1'
+	compile 'org.apache.logging.log4j:log4j-core:2.9.1'
-	compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.9.1'
+	compile 'org.apache.logging.log4j:log4j-slf4j-impl:2.9.1'
-}
+}