add percentile plots

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java

@@ -5,10 +5,12 @@ import java.io.File;
 public class AggregatedData {
 	private final String label;
 	private File dataFile;
+	private double average;
 
-	public AggregatedData(String label, File dataFile) {
+	public AggregatedData(String label, File dataFile, double average) {
 		this.label = label;
 		this.dataFile = dataFile;
+		this.average = average;
 	}
 
 	public String getLabel() {
@@ -18,4 +20,8 @@ public class AggregatedData {
 	public File getDataFile() {
 		return dataFile;
 	}
+	
+	public double getAverage() {
+		return average;
+	}
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java

@@ -13,7 +13,7 @@ import org.lucares.collections.IntList;
 
 public class PercentileCustomAggregator implements CustomAggregator{
 
-	private final static int POINTS = 300;
+	private final static int POINTS = 100;
 	
 	private final IntList values = new IntList(); // TODO should be a LongList
 
@@ -37,7 +37,7 @@ public class PercentileCustomAggregator implements CustomAggregator{
 		
 		values.parallelSort();
 		
-		
+		final IntList percentiles = new IntList(POINTS);
 		final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
 		try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
 			
@@ -47,9 +47,12 @@ public class PercentileCustomAggregator implements CustomAggregator{
 				for (int i = 0; i < POINTS; i++) {
 					data.append(i* (100/(double)POINTS));
 					data.append(separator);
-					data.append(values.get((int) Math.floor(values.size()
+					int percentile = values.get((int) Math.floor(values.size()
-							/ ((double)POINTS) * i)));
+							/ ((double)POINTS) * i));
+					data.append(percentile);
 					data.append(newline);
+					
+					percentiles.add(percentile);
 				}
 				final int maxValue = values.get(values.size() - 1);
 				data.append(100);
@@ -58,11 +61,14 @@ public class PercentileCustomAggregator implements CustomAggregator{
 				data.append(newline);
 			}
 			output.write(data.toString());
+			
 		}
+		// TODO remove:
+		double average = percentiles.stream().summaryStatistics().getAverage();
 		
 		
 		final String title = String.format("percentiles");
-		return new AggregatedData(title, dataFile);
+		return new AggregatedData(title, dataFile, average);
 	}
 
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java

@@ -19,6 +19,10 @@ public interface DataSeries {
 		return result < 0 ? -1 : (result > 0 ? 1 : 0);
 	};
 	
+	public static final Comparator<? super DataSeries> BY_NAME = (a,b) -> {
+		return a.getTitle().compareToIgnoreCase(b.getTitle());
+	};
+
 	public String getIdAsString();
 	
 	public int getId();
@@ -58,7 +62,7 @@ public interface DataSeries {
 		case MIN_VALUE:
 			return DataSeries.BY_MAX_VALUE;
 		case NO_LIMIT:
-			return DataSeries.BY_NUMBER_OF_VALUES;
+			return DataSeries.BY_NAME;
 		}
 		throw new IllegalStateException("unhandled enum: "+ limitBy);
 	}
@@ -89,10 +93,13 @@ public interface DataSeries {
 			final int numColors = GnuplotColorPalettes.DEFAULT.size();
 			final int numDashTypes = DashTypes.DEFAULT.size();
 			
-			final String color = GnuplotColorPalettes.DEFAULT.get(i % numColors).getColor();
+			GnuplotColor color = GnuplotColorPalettes.DEFAULT.get(i % numColors);
+			if (dataSerie.getAggregatedData() != null){
+			//	color = color.brighter();
+			}
 			final String dashType = DashTypes.DEFAULT.get((i/numColors) % numDashTypes); 
 			String style = String.format("lt %s dt %s ",// 
-					color,//
+					color.getColor(),//
 					dashType//
 					);
 			dataSerie.setStyle(style);

pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java

@@ -65,9 +65,14 @@ public class FileBackedDataSeries implements DataSeries {
 
 	@Override
 	public String getGnuplotPlotDefinition() {
+		String average="";
+		if (getAggregatedData() != null){
+			average = String.format(" [%.2f]", getAggregatedData().getAverage());
+		}
+		
 		return String.format("'%s' using 1:2 title '%s' with %s %s, \\", //
 				getDataFile(),//
-				getTitle(),//
+				getTitle()+average,//
 				linetype, // line or points
 				style//
 				);

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotAxis.java

@@ -0,0 +1,22 @@
+package org.lucares.recommind.logs;
+
+public enum GnuplotAxis {
+	X1Y1("x1y1"), 
+	
+	X1Y2("x1y2"), 
+	
+	X2Y1("x2y1"), 
+	
+	X2Y2("x2y2");
+	
+	private String axis;
+
+	private GnuplotAxis(String axis) {
+		this.axis = axis;
+	}
+	
+	@Override
+	public String toString() {
+		return axis;
+	}
+}

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColor.java

@@ -1,26 +1,56 @@
 package org.lucares.recommind.logs;
 
+import java.awt.Color;
+
 public class GnuplotColor {
-	private final String color; // either a name ('darkorchid') or hex ('#00efcc')
+	private final String color; // hex: 00efcc
 
 	private GnuplotColor(String color) {
 		this.color = color;
 	}
 
-	public static GnuplotColor byHex(String aHex){
+	public static GnuplotColor byHex(String aHex) {
-		return new GnuplotColor("rgb \"" + aHex+"\"");
+		return new GnuplotColor(aHex);
 	}
 
-	public static GnuplotColor byName(GnuplotColorNames aName){
+	public static GnuplotColor byAwtColor(Color color) {
-		return new GnuplotColor(aName.getColor());
+
+		final String hex = String.format("%02x%02x%02x",//
+				color.getRed(),//
+				color.getGreen(),//
+				color.getBlue()//
+				);
+
+		return new GnuplotColor(hex);
 	}
 
 	public String getColor() {
-		return color;
+		return "rgb \"#" + color + "\"";
 	}
 
 	@Override
 	public String toString() {
-		return color;
+		return getColor();
+	}
+
+	Color toAwtColor() {
+		int red = Integer.parseInt(color.substring(0, 2), 16);
+		int green = Integer.parseInt(color.substring(2, 4), 16);
+		int blue = Integer.parseInt(color.substring(4, 6), 16);
+		return new Color(red, green, blue);
+	}
+
+	public GnuplotColor brighter() {
+
+		final Color brighterColor = toAwtColor().brighter();
+		
+		
+		return byAwtColor(brighterColor);
+	}
+
+	public GnuplotColor darker() {
+
+		final Color darkerColor = toAwtColor().darker();
+		return byAwtColor(darkerColor);
 	}
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorNames.java

@@ -1,300 +0,0 @@
-package org.lucares.recommind.logs;
-
-public enum GnuplotColorNames {
-
-	ALICEBLUE("aliceblue"),
-
-	ANTIQUEWHITE("antiquewhite"),
-
-	AQUA("aqua"),
-
-	AQUAMARINE("aquamarine"),
-
-	AZURE("azure"),
-
-	BEIGE("beige"),
-
-	BISQUE("bisque"),
-
-	BLACK("black"),
-
-	BLANCHEDALMOND("blanchedalmond"),
-
-	BLUE("blue"),
-
-	BLUEVIOLET("blueviolet"),
-
-	BROWN("brown"),
-
-	BURLYWOOD("burlywood"),
-
-	CADETBLUE("cadetblue"),
-
-	CHARTREUSE("chartreuse"),
-
-	CHOCOLATE("chocolate"),
-
-	CORAL("coral"),
-
-	CORNFLOWERBLUE("cornflowerblue"),
-
-	CORNSILK("cornsilk"),
-
-	CRIMSON("crimson"),
-
-	CYAN("cyan"),
-
-	DARKBLUE("darkblue"),
-
-	DARKCYAN("darkcyan"),
-
-	DARKGOLDENROD("darkgoldenrod"),
-
-	DARKGRAY("darkgray"),
-
-	DARKGREEN("darkgreen"),
-
-	DARKKHAKI("darkkhaki"),
-
-	DARKMAGENTA("darkmagenta"),
-
-	DARKOLIVEGREEN("darkolivegreen"),
-
-	DARKORANGE("darkorange"),
-
-	DARKORCHID("darkorchid"),
-
-	DARKRED("darkred"),
-
-	DARKSALMON("darksalmon"),
-
-	DARKSEAGREEN("darkseagreen"),
-
-	DARKSLATEBLUE("darkslateblue"),
-
-	DARKSLATEGRAY("darkslategray"),
-
-	DARKTURQUOISE("darkturquoise"),
-
-	DARKVIOLET("darkviolet"),
-
-	DEEPPINK("deeppink"),
-
-	DEEPSKYBLUE("deepskyblue"),
-
-	DIMGRAY("dimgray"),
-
-	DODGERBLUE("dodgerblue"),
-
-	FIREBRICK("firebrick"),
-
-	FLORALWHITE("floralwhite"),
-
-	FORESTGREEN("forestgreen"),
-
-	FUCHSIA("fuchsia"),
-
-	GAINSBORO("gainsboro"),
-
-	GHOSTWHITE("ghostwhite"),
-
-	GOLD("gold"),
-
-	GOLDENROD("goldenrod"),
-
-	GRAY("gray"),
-
-	GREEN("green"),
-
-	GREENYELLOW("greenyellow"),
-
-	HONEYDEW("honeydew"),
-
-	HOTPINK("hotpink"),
-
-	INDIANRED("indianred"),
-
-	INDIGO("indigo"),
-
-	IVORY("ivory"),
-
-	KHAKI("khaki"),
-
-	LAVENDER("lavender"),
-
-	LAVENDERBLUSH("lavenderblush"),
-
-	LAWNGREEN("lawngreen"),
-
-	LEMONCHIFFON("lemonchiffon"),
-
-	LIGHTBLUE("lightblue"),
-
-	LIGHTCORAL("lightcoral"),
-
-	LIGHTCYAN("lightcyan"),
-
-	LIGHTGOLDENRODYE("lightgoldenrodye"),
-
-	LIGHTGREEN("lightgreen"),
-
-	LIGHTGREY("lightgrey"),
-
-	LIGHTPINK("lightpink"),
-
-	LIGHTSALMON("lightsalmon"),
-
-	LIGHTSEAGREEN("lightseagreen"),
-
-	LIGHTSKYBLUE("lightskyblue"),
-
-	LIGHTSLATEGRAY("lightslategray"),
-
-	LIGHTSTEELBLUE("lightsteelblue"),
-
-	LIGHTYELLOW("lightyellow"),
-
-	LIME("lime"),
-
-	LIMEGREEN("limegreen"),
-
-	LINEN("linen"),
-
-	MAGENTA("magenta"),
-
-	MAROON("maroon"),
-
-	MEDIUMAQUAMARINE("mediumaquamarine"),
-
-	MEDIUMBLUE("mediumblue"),
-
-	MEDIUMORCHID("mediumorchid"),
-
-	MEDIUMPURPLE("mediumpurple"),
-
-	MEDIUMSEAGREEN("mediumseagreen"),
-
-	MEDIUMSLATEBLUE("mediumslateblue"),
-
-	MEDIUMSPRINGGREE("mediumspringgree"),
-
-	MEDIUMTURQUOISE("mediumturquoise"),
-
-	MEDIUMVIOLETRED("mediumvioletred"),
-
-	MIDNIGHTBLUE("midnightblue"),
-
-	MINTCREAM("mintcream"),
-
-	MISTYROSE("mistyrose"),
-
-	MOCCASIN("moccasin"),
-
-	NAVAJOWHITE("navajowhite"),
-
-	NAVY("navy"),
-
-	NAVYBLUE("navyblue"),
-
-	OLDLACE("oldlace"),
-
-	OLIVE("olive"),
-
-	OLIVEDRAB("olivedrab"),
-
-	ORANGE("orange"),
-
-	ORANGERED("orangered"),
-
-	ORCHID("orchid"),
-
-	PALEGOLDENROD("palegoldenrod"),
-
-	PALEGREEN("palegreen"),
-
-	PALETURQUOISE("paleturquoise"),
-
-	PALEVIOLETRED("palevioletred"),
-
-	PAPAYAWHIP("papayawhip"),
-
-	PEACHPUFF("peachpuff"),
-
-	PERU("peru"),
-
-	PINK("pink"),
-
-	PLUM("plum"),
-
-	POWDERBLUE("powderblue"),
-
-	PURPLE("purple"),
-
-	RED("red"),
-
-	ROSYBROWN("rosybrown"),
-
-	ROYALBLUE("royalblue"),
-
-	SADDLEBROWN("saddlebrown"),
-
-	SALMON("salmon"),
-
-	SANDYBROWN("sandybrown"),
-
-	SEAGREEN("seagreen"),
-
-	SEASHELL("seashell"),
-
-	SIENNA("sienna"),
-
-	SILVER("silver"),
-
-	SKYBLUE("skyblue"),
-
-	SLATEBLUE("slateblue"),
-
-	SLATEGRAY("slategray"),
-
-	SNOW("snow"),
-
-	SPRINGGREEN("springgreen"),
-
-	STEELBLUE("steelblue"),
-
-	TAN("tan"),
-
-	TEAL("teal"),
-
-	THISTLE("thistle"),
-
-	TOMATO("tomato"),
-
-	TURQUOISE("turquoise"),
-
-	VIOLET("violet"),
-
-	WHEAT("wheat"),
-
-	WHITE("white"),
-
-	WHITESMOKE("whitesmoke"),
-
-	YELLOW("yellow"),
-
-	YELLOWGREEN("yellowgreen");
-
-	private final String color;
-
-	private GnuplotColorNames(final String color) {
-		this.color = color;
-	}
-
-	public String getColor() {
-		return color;
-	}
-	
-	public GnuplotColor toColor(){
-		return GnuplotColor.byName(this);
-	}
-}

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java

@@ -21,16 +21,16 @@ public interface GnuplotColorPalettes {
 	 */
 	
 	List<GnuplotColor> DEFAULT = Arrays.asList(
-			GnuplotColor.byHex("#9400D3"),//
+			GnuplotColor.byHex("9400D3"),//
-			GnuplotColor.byHex("#009e73"),//
+			GnuplotColor.byHex("009e73"),//
-			GnuplotColor.byHex("#56b4e9"),//
+			GnuplotColor.byHex("56b4e9"),//
-			GnuplotColor.byHex("#e69f00"),//
+			GnuplotColor.byHex("e69f00"),//
-			GnuplotColor.byHex("#f0e442"),//
+			GnuplotColor.byHex("f0e442"),//
-			GnuplotColor.byHex("#0072b2"),//
+			GnuplotColor.byHex("0072b2"),//
-			GnuplotColor.byHex("#e51e10"),//
+			GnuplotColor.byHex("e51e10"),//
-			GnuplotColor.byHex("#0000FF"),//
+			GnuplotColor.byHex("0000FF"),//
-			GnuplotColor.byHex("#FF69B4"),//
+			GnuplotColor.byHex("FF69B4"),//
-			GnuplotColor.byHex("#00BFFF"),//
+			GnuplotColor.byHex("00BFFF"),//
-			GnuplotColorNames.BLACK.toColor()//
+			GnuplotColor.byHex("000000")//
 			); 
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -27,10 +27,10 @@ public class GnuplotFileGenerator {
 		}
 		appendfln(result, "set format x \"%s\"", xAxis.getFormatX());
 		appendfln(result, "set xlabel \"%s\"", xAxis.getXlabel());
-		appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel());
+		appendfln(result, "set xtics nomirror rotate by %d", xAxis.getRotateXAxisLabel());
 		appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo());
 		
-		appendfln(result, "set x2label \"percentile\"");
+		appendfln(result, "set x2label \"Percentile\"");
 		appendln(result, "set format x2 \"%.0f%%\"");
 		appendfln(result, "set x2tics");
 		appendfln(result, "set x2range [\"0\":\"100\"]");

pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java

@@ -98,6 +98,7 @@ public class PercentilePlot implements ConcretePlotter {
 			final Limit limitBy = plotSettings.getLimitBy();
 			int limit = plotSettings.getLimit();
 			DataSeries.sortAndLimit(dataSeries, limitBy, limit);
+			DataSeries.setColors(dataSeries);
 
 			final Path outputFile = Files.createTempFile(outputDir, "out",
 					".png");

pdb-ui/src/main/resources/application-testing.properties

@@ -0,0 +1,4 @@
+#db.base=D:/ws/pdb/dataNew
+db.base=C:/Temp/pdb/testing
+server.port=17333
+gnuplot.home=D:/ws/pdb/gnuplot-5.2

pdb-ui/src/main/resources/log4j2.xml

@@ -38,8 +38,8 @@
   	<logger name="org.lucares.metrics.proposals" level="DEBUG" />
   	<logger name="org.lucares.metrics.plotter" level="DEBUG" />
   	<logger name="org.lucares.metrics.gnuplot" level="DEBUG" />
-  	<logger name="org.lucares.metrics.ingestion.tagsToFile.newPdbWriter" level="DEBUG" />
   	<!-- 
+  	<logger name="org.lucares.metrics.ingestion.tagsToFile.newPdbWriter" level="DEBUG" />
   	<logger name="org.lucares.metrics.dataStore" level="DEBUG" />
   	<logger name="org.lucares.pdb.datastore.lang.QueryCompletionPdbLangParser" level="TRACE" />
   	<logger name="org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor" level="TRACE" />

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -95,8 +95,8 @@ public class PerformanceDb implements AutoCloseable {
 						final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
 
 						METRICS_LOGGER
-								.debug(String.format("inserting %d/s ; the last %,d took %dms; total entries: %,d; last entry: %s",
+								.debug(String.format("inserting %d/s ; total: %,d; last: %s",
-										entriesPerSecond, insertionsSinceLastSync, duration, count, entry));
+										entriesPerSecond, count, entry));
 						tagsToFile.flush();
 
 						lastSync = System.currentTimeMillis();