andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add percentile plots

Browse Source
This commit is contained in:
ahr
2018-03-03 08:19:26 +01:00
parent b4a0514267
commit 6b60fd542c
13 changed files with 118 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java
View File

@@ -5,10 +5,12 @@ import java.io.File;
public class AggregatedData {
private final String label;
private File dataFile;
private double average;
public AggregatedData(String label, File dataFile) {
public AggregatedData(String label, File dataFile, double average) {
this.label = label;
this.dataFile = dataFile;
this.average = average;
}
public String getLabel() {
@@ -18,4 +20,8 @@ public class AggregatedData {
public File getDataFile() {
return dataFile;
}
public double getAverage() {
return average;
}
}

16
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java
View File

@@ -13,7 +13,7 @@ import org.lucares.collections.IntList;
public class PercentileCustomAggregator implements CustomAggregator{
private final static int POINTS = 300;
private final static int POINTS = 100;
private final IntList values = new IntList(); // TODO should be a LongList
@@ -37,7 +37,7 @@ public class PercentileCustomAggregator implements CustomAggregator{
values.parallelSort();
final IntList percentiles = new IntList(POINTS);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
@@ -47,9 +47,12 @@ public class PercentileCustomAggregator implements CustomAggregator{
for (int i = 0; i < POINTS; i++) {
data.append(i* (100/(double)POINTS));
data.append(separator);
data.append(values.get((int) Math.floor(values.size()
/ ((double)POINTS) * i)));
int percentile = values.get((int) Math.floor(values.size()
/ ((double)POINTS) * i));
data.append(percentile);
data.append(newline);
percentiles.add(percentile);
}
final int maxValue = values.get(values.size() - 1);
data.append(100);
@@ -58,11 +61,14 @@ public class PercentileCustomAggregator implements CustomAggregator{
data.append(newline);
}
output.write(data.toString());
}
// TODO remove:
double average = percentiles.stream().summaryStatistics().getAverage();
final String title = String.format("percentiles");
return new AggregatedData(title, dataFile);
return new AggregatedData(title, dataFile, average);
}
}

13
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -19,6 +19,10 @@ public interface DataSeries {
return result < 0 ? -1 : (result > 0 ? 1 : 0);
};
public static final Comparator<? super DataSeries> BY_NAME = (a,b) -> {
return a.getTitle().compareToIgnoreCase(b.getTitle());
};
public String getIdAsString();
public int getId();
@@ -58,7 +62,7 @@ public interface DataSeries {
case MIN_VALUE:
return DataSeries.BY_MAX_VALUE;
case NO_LIMIT:
return DataSeries.BY_NUMBER_OF_VALUES;
return DataSeries.BY_NAME;
}
throw new IllegalStateException("unhandled enum: "+ limitBy);
}
@@ -89,10 +93,13 @@ public interface DataSeries {
final int numColors = GnuplotColorPalettes.DEFAULT.size();
final int numDashTypes = DashTypes.DEFAULT.size();
final String color = GnuplotColorPalettes.DEFAULT.get(i % numColors).getColor();
GnuplotColor color = GnuplotColorPalettes.DEFAULT.get(i % numColors);
if (dataSerie.getAggregatedData() != null){
// color = color.brighter();
}
final String dashType = DashTypes.DEFAULT.get((i/numColors) % numDashTypes);
String style = String.format("lt %s dt %s ",//
color,//
color.getColor(),//
dashType//
);
dataSerie.setStyle(style);

7
pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java
View File

@@ -65,9 +65,14 @@ public class FileBackedDataSeries implements DataSeries {
@Override
public String getGnuplotPlotDefinition() {
String average="";
if (getAggregatedData() != null){
average = String.format(" [%.2f]", getAggregatedData().getAverage());
}
return String.format("'%s' using 1:2 title '%s' with %s %s, \\", //
getDataFile(),//
getTitle(),//
getTitle()+average,//
linetype, // line or points
style//
);

22
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotAxis.java Normal file
View File

@@ -0,0 +1,22 @@
package org.lucares.recommind.logs;
public enum GnuplotAxis {
X1Y1("x1y1"),
X1Y2("x1y2"),
X2Y1("x2y1"),
X2Y2("x2y2");
private String axis;
private GnuplotAxis(String axis) {
this.axis = axis;
}
@Override
public String toString() {
return axis;
}
}

42
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColor.java
View File

@@ -1,26 +1,56 @@
package org.lucares.recommind.logs;
import java.awt.Color;
public class GnuplotColor {
private final String color; // either a name ('darkorchid') or hex ('#00efcc')
private final String color; // hex: 00efcc
private GnuplotColor(String color) {
this.color = color;
}
public static GnuplotColor byHex(String aHex) {
return new GnuplotColor("rgb \"" + aHex+"\"");
return new GnuplotColor(aHex);
}
public static GnuplotColor byName(GnuplotColorNames aName){
return new GnuplotColor(aName.getColor());
public static GnuplotColor byAwtColor(Color color) {
final String hex = String.format("%02x%02x%02x",//
color.getRed(),//
color.getGreen(),//
color.getBlue()//
);
return new GnuplotColor(hex);
}
public String getColor() {
return color;
return "rgb \"#" + color + "\"";
}
@Override
public String toString() {
return color;
return getColor();
}
Color toAwtColor() {
int red = Integer.parseInt(color.substring(0, 2), 16);
int green = Integer.parseInt(color.substring(2, 4), 16);
int blue = Integer.parseInt(color.substring(4, 6), 16);
return new Color(red, green, blue);
}
public GnuplotColor brighter() {
final Color brighterColor = toAwtColor().brighter();
return byAwtColor(brighterColor);
}
public GnuplotColor darker() {
final Color darkerColor = toAwtColor().darker();
return byAwtColor(darkerColor);
}
}

300
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorNames.java
View File

@@ -1,300 +0,0 @@
package org.lucares.recommind.logs;
public enum GnuplotColorNames {
ALICEBLUE("aliceblue"),
ANTIQUEWHITE("antiquewhite"),
AQUA("aqua"),
AQUAMARINE("aquamarine"),
AZURE("azure"),
BEIGE("beige"),
BISQUE("bisque"),
BLACK("black"),
BLANCHEDALMOND("blanchedalmond"),
BLUE("blue"),
BLUEVIOLET("blueviolet"),
BROWN("brown"),
BURLYWOOD("burlywood"),
CADETBLUE("cadetblue"),
CHARTREUSE("chartreuse"),
CHOCOLATE("chocolate"),
CORAL("coral"),
CORNFLOWERBLUE("cornflowerblue"),
CORNSILK("cornsilk"),
CRIMSON("crimson"),
CYAN("cyan"),
DARKBLUE("darkblue"),
DARKCYAN("darkcyan"),
DARKGOLDENROD("darkgoldenrod"),
DARKGRAY("darkgray"),
DARKGREEN("darkgreen"),
DARKKHAKI("darkkhaki"),
DARKMAGENTA("darkmagenta"),
DARKOLIVEGREEN("darkolivegreen"),
DARKORANGE("darkorange"),
DARKORCHID("darkorchid"),
DARKRED("darkred"),
DARKSALMON("darksalmon"),
DARKSEAGREEN("darkseagreen"),
DARKSLATEBLUE("darkslateblue"),
DARKSLATEGRAY("darkslategray"),
DARKTURQUOISE("darkturquoise"),
DARKVIOLET("darkviolet"),
DEEPPINK("deeppink"),
DEEPSKYBLUE("deepskyblue"),
DIMGRAY("dimgray"),
DODGERBLUE("dodgerblue"),
FIREBRICK("firebrick"),
FLORALWHITE("floralwhite"),
FORESTGREEN("forestgreen"),
FUCHSIA("fuchsia"),
GAINSBORO("gainsboro"),
GHOSTWHITE("ghostwhite"),
GOLD("gold"),
GOLDENROD("goldenrod"),
GRAY("gray"),
GREEN("green"),
GREENYELLOW("greenyellow"),
HONEYDEW("honeydew"),
HOTPINK("hotpink"),
INDIANRED("indianred"),
INDIGO("indigo"),
IVORY("ivory"),
KHAKI("khaki"),
LAVENDER("lavender"),
LAVENDERBLUSH("lavenderblush"),
LAWNGREEN("lawngreen"),
LEMONCHIFFON("lemonchiffon"),
LIGHTBLUE("lightblue"),
LIGHTCORAL("lightcoral"),
LIGHTCYAN("lightcyan"),
LIGHTGOLDENRODYE("lightgoldenrodye"),
LIGHTGREEN("lightgreen"),
LIGHTGREY("lightgrey"),
LIGHTPINK("lightpink"),
LIGHTSALMON("lightsalmon"),
LIGHTSEAGREEN("lightseagreen"),
LIGHTSKYBLUE("lightskyblue"),
LIGHTSLATEGRAY("lightslategray"),
LIGHTSTEELBLUE("lightsteelblue"),
LIGHTYELLOW("lightyellow"),
LIME("lime"),
LIMEGREEN("limegreen"),
LINEN("linen"),
MAGENTA("magenta"),
MAROON("maroon"),
MEDIUMAQUAMARINE("mediumaquamarine"),
MEDIUMBLUE("mediumblue"),
MEDIUMORCHID("mediumorchid"),
MEDIUMPURPLE("mediumpurple"),
MEDIUMSEAGREEN("mediumseagreen"),
MEDIUMSLATEBLUE("mediumslateblue"),
MEDIUMSPRINGGREE("mediumspringgree"),
MEDIUMTURQUOISE("mediumturquoise"),
MEDIUMVIOLETRED("mediumvioletred"),
MIDNIGHTBLUE("midnightblue"),
MINTCREAM("mintcream"),
MISTYROSE("mistyrose"),
MOCCASIN("moccasin"),
NAVAJOWHITE("navajowhite"),
NAVY("navy"),
NAVYBLUE("navyblue"),
OLDLACE("oldlace"),
OLIVE("olive"),
OLIVEDRAB("olivedrab"),
ORANGE("orange"),
ORANGERED("orangered"),
ORCHID("orchid"),
PALEGOLDENROD("palegoldenrod"),
PALEGREEN("palegreen"),
PALETURQUOISE("paleturquoise"),
PALEVIOLETRED("palevioletred"),
PAPAYAWHIP("papayawhip"),
PEACHPUFF("peachpuff"),
PERU("peru"),
PINK("pink"),
PLUM("plum"),
POWDERBLUE("powderblue"),
PURPLE("purple"),
RED("red"),
ROSYBROWN("rosybrown"),
ROYALBLUE("royalblue"),
SADDLEBROWN("saddlebrown"),
SALMON("salmon"),
SANDYBROWN("sandybrown"),
SEAGREEN("seagreen"),
SEASHELL("seashell"),
SIENNA("sienna"),
SILVER("silver"),
SKYBLUE("skyblue"),
SLATEBLUE("slateblue"),
SLATEGRAY("slategray"),
SNOW("snow"),
SPRINGGREEN("springgreen"),
STEELBLUE("steelblue"),
TAN("tan"),
TEAL("teal"),
THISTLE("thistle"),
TOMATO("tomato"),
TURQUOISE("turquoise"),
VIOLET("violet"),
WHEAT("wheat"),
WHITE("white"),
WHITESMOKE("whitesmoke"),
YELLOW("yellow"),
YELLOWGREEN("yellowgreen");
private final String color;
private GnuplotColorNames(final String color) {
this.color = color;
}
public String getColor() {
return color;
}
public GnuplotColor toColor(){
return GnuplotColor.byName(this);
}
}

22
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java
View File

@@ -21,16 +21,16 @@ public interface GnuplotColorPalettes {
*/
List<GnuplotColor> DEFAULT = Arrays.asList(
GnuplotColor.byHex("#9400D3"),//
GnuplotColor.byHex("#009e73"),//
GnuplotColor.byHex("#56b4e9"),//
GnuplotColor.byHex("#e69f00"),//
GnuplotColor.byHex("#f0e442"),//
GnuplotColor.byHex("#0072b2"),//
GnuplotColor.byHex("#e51e10"),//
GnuplotColor.byHex("#0000FF"),//
GnuplotColor.byHex("#FF69B4"),//
GnuplotColor.byHex("#00BFFF"),//
GnuplotColorNames.BLACK.toColor()//
GnuplotColor.byHex("9400D3"),//
GnuplotColor.byHex("009e73"),//
GnuplotColor.byHex("56b4e9"),//
GnuplotColor.byHex("e69f00"),//
GnuplotColor.byHex("f0e442"),//
GnuplotColor.byHex("0072b2"),//
GnuplotColor.byHex("e51e10"),//
GnuplotColor.byHex("0000FF"),//
GnuplotColor.byHex("FF69B4"),//
GnuplotColor.byHex("00BFFF"),//
GnuplotColor.byHex("000000")//
);
}

4
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -27,10 +27,10 @@ public class GnuplotFileGenerator {
}
appendfln(result, "set format x \"%s\"", xAxis.getFormatX());
appendfln(result, "set xlabel \"%s\"", xAxis.getXlabel());
appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel());
appendfln(result, "set xtics nomirror rotate by %d", xAxis.getRotateXAxisLabel());
appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo());
appendfln(result, "set x2label \"percentile\"");
appendfln(result, "set x2label \"Percentile\"");
appendln(result, "set format x2 \"%.0f%%\"");
appendfln(result, "set x2tics");
appendfln(result, "set x2range [\"0\":\"100\"]");

1
pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java
View File

@@ -98,6 +98,7 @@ public class PercentilePlot implements ConcretePlotter {
final Limit limitBy = plotSettings.getLimitBy();
int limit = plotSettings.getLimit();
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
DataSeries.setColors(dataSeries);
final Path outputFile = Files.createTempFile(outputDir, "out",
".png");

4
pdb-ui/src/main/resources/application-testing.properties Normal file
View File

@@ -0,0 +1,4 @@
#db.base=D:/ws/pdb/dataNew
db.base=C:/Temp/pdb/testing
server.port=17333
gnuplot.home=D:/ws/pdb/gnuplot-5.2

2
pdb-ui/src/main/resources/log4j2.xml
View File

@@ -38,8 +38,8 @@
<logger name="org.lucares.metrics.proposals" level="DEBUG" />
<logger name="org.lucares.metrics.plotter" level="DEBUG" />
<logger name="org.lucares.metrics.gnuplot" level="DEBUG" />
<logger name="org.lucares.metrics.ingestion.tagsToFile.newPdbWriter" level="DEBUG" />
<!--
<logger name="org.lucares.metrics.ingestion.tagsToFile.newPdbWriter" level="DEBUG" />
<logger name="org.lucares.metrics.dataStore" level="DEBUG" />
<logger name="org.lucares.pdb.datastore.lang.QueryCompletionPdbLangParser" level="TRACE" />
<logger name="org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor" level="TRACE" />

4
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -95,8 +95,8 @@ public class PerformanceDb implements AutoCloseable {
final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
METRICS_LOGGER
.debug(String.format("inserting %d/s ; the last %,d took %dms; total entries: %,d; last entry: %s",
entriesPerSecond, insertionsSinceLastSync, duration, count, entry));
.debug(String.format("inserting %d/s ; total: %,d; last: %s",
entriesPerSecond, count, entry));
tagsToFile.flush();
lastSync = System.currentTimeMillis();