andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

render plot with a single dataseriew

Browse Source
This commit is contained in:
Andreas Huber
2016-12-10 18:50:29 +01:00
parent 81b39c5675
commit e936df6f7e
13 changed files with 651 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
performanceDb/src/main/java/org/lucares/performance/db/Fields.java
View File

@@ -1,23 +1,11 @@
package org.lucares.performance.db;
class Fields {
static final String TAG_PREFIX = "__tag_";
static final String PREFIX_INTERNAL_TAG = "__internal_";
static final String DATE_OFFSET = "__date_offset__";
static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset";
static String prefixedKey(final String key) {
return TAG_PREFIX + key;
static boolean isInternalField(final String key) {
return key.startsWith(PREFIX_INTERNAL_TAG);
}
static boolean isPrefixedKey(final String key) {
return key.startsWith(TAG_PREFIX);
}
static String stripPrefix(final String key) {
if (!isPrefixedKey(key)) {
throw new IllegalArgumentException(key + " is not prefixed by " + TAG_PREFIX);
}
return key.substring(TAG_PREFIX.length());
}
}

3
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -31,6 +31,9 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader == null) {
nextFile();
}
if (reader == null) {
return null;
}
final Optional<Entry> optionalEntry = reader.readEntry(currentPdbFile.getTags());
return optionalEntry.orElseGet(() -> {

15
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -115,7 +115,22 @@ public class PerformanceDb implements AutoCloseable {
public Stream<Entry> get(final Tags tags) {
final List<PdbFile> pdbFiles = tagsToFile.getFilesMatchingTags(tags);
return toStream(pdbFiles);
}
/**
* Return the entries as an unbound, ordered and non-parallel stream.
*
* @param query
* @return {@link Stream} unbound, ordered and non-parallel
*/
public Stream<Entry> get(final String query) {
final List<PdbFile> pdbFiles = tagsToFile.getFilesForQuery(query);
return toStream(pdbFiles);
}
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final Iterator<Entry> iterator = new PdbFileIterator(pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);

2
performanceDb/src/main/java/org/lucares/performance/db/Query.java
View File

@@ -6,7 +6,7 @@ final class Query {
for (final String key : tags.getKeys()) {
tags.getValue(key).ifPresent(value -> {
result.append(Fields.prefixedKey(key));
result.append(key);
result.append("=");
result.append(value);
result.append(" ");

9
performanceDb/src/main/java/org/lucares/performance/db/Tags.java
View File

@@ -21,6 +21,15 @@ public class Tags {
private Tags(final Map<String, String> tags) {
this.tags.putAll(tags);
ensureNoInternalFields();
}
private void ensureNoInternalFields() {
tags.keySet().forEach(key -> {
if (Fields.isInternalField(key)) {
throw new IllegalArgumentException(key + " is an internal field. Choose another prefix.");
}
});
}
public static Tags create() {

18
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -48,8 +48,12 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
}
List<PdbFile> getFilesMatchingTags(final Tags tags) {
final List<PdbFile> result = new ArrayList<>();
final String query = Query.createQuery(tags);
return getFilesForQuery(query);
}
List<PdbFile> getFilesForQuery(final String query) {
final List<PdbFile> result = new ArrayList<>();
try {
final List<Document> searchResult = db.search(query);
@@ -81,9 +85,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
for (final String key : document.getProperties().keySet()) {
if (Fields.isPrefixedKey(key)) {
if (!Fields.isInternalField(key)) {
final String value = document.getPropertyString(key);
tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value);
tagsOfFile = tagsOfFile.copyAdd(key, value);
}
}
return tagsOfFile;
@@ -130,7 +134,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
ensureFieldsExist(tags);
tags.forEach((key, value) -> {
db.setProperty(file, Fields.prefixedKey(key), value);
db.setProperty(file, key, value);
});
db.setProperty(file, Fields.DATE_OFFSET, day.serialize());
@@ -145,11 +149,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
tags.forEach((key, value) -> {
final String prefixedKey = Fields.prefixedKey(key);
final Field field = fieldsMap.get(prefixedKey);
final Field field = fieldsMap.get(key);
if (field == null) {
db.createField(prefixedKey, FieldType.STRING);
db.createField(key, FieldType.STRING);
}
});
}

1
recommind-logs/build.gradle
View File

@@ -2,5 +2,6 @@
dependencies {
compile project(':performanceDb')
compile "io.thekraken:grok:0.1.5"
compile 'org.lucares:svak:1.0'
}

45
recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java Normal file
View File

@@ -0,0 +1,45 @@
package org.lucares.recommind.logs;
import java.io.File;
public class DataSeries {
private final File dataFile;
private final String title;
private final GnuplotColor color;
private final Integer pointType;
public DataSeries(final File dataFile, final String title) {
super();
this.dataFile = dataFile;
this.title = title;
this.color = null;
this.pointType = null;
}
public DataSeries(final File dataFile, final String title, final GnuplotColor color, final Integer pointType) {
super();
this.dataFile = dataFile;
this.title = title;
this.color = color;
this.pointType = pointType;
}
public GnuplotColor getColor() {
return color;
}
public Integer getPointType() {
return pointType;
}
public File getDataFile() {
return dataFile;
}
public String getTitle() {
return title;
}
}

36
recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java Normal file
View File

@@ -0,0 +1,36 @@
package org.lucares.recommind.logs;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Collection;
import com.google.common.io.Files;
public class Gnuplot {
private final Path tmpDirectory;
public Gnuplot(final Path tmpDirectory) {
this.tmpDirectory = tmpDirectory;
}
public void plot(final GnuplotSettings settings, final Collection<DataSeries> dataSeries)
throws IOException, InterruptedException {
final GnuplotFileGenerator generator = new GnuplotFileGenerator();
final String gnuplotFileContent = generator.generate(settings, dataSeries);
System.out.println(gnuplotFileContent);
final File gnuplotFile = File.createTempFile("gnuplot", ".dem", tmpDirectory.toFile());
Files.write(gnuplotFileContent, gnuplotFile, StandardCharsets.UTF_8);
final ProcessBuilder processBuilder = new ProcessBuilder("gnuplot", gnuplotFile.getAbsolutePath());
processBuilder.inheritIO();
final Process start = processBuilder.start();
start.waitFor();
}
}

296
recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java Normal file
View File

@@ -0,0 +1,296 @@
package org.lucares.recommind.logs;
public enum GnuplotColor {
ALICEBLUE("aliceblue"),
ANTIQUEWHITE("antiquewhite"),
AQUA("aqua"),
AQUAMARINE("aquamarine"),
AZURE("azure"),
BEIGE("beige"),
BISQUE("bisque"),
BLACK("black"),
BLANCHEDALMOND("blanchedalmond"),
BLUE("blue"),
BLUEVIOLET("blueviolet"),
BROWN("brown"),
BURLYWOOD("burlywood"),
CADETBLUE("cadetblue"),
CHARTREUSE("chartreuse"),
CHOCOLATE("chocolate"),
CORAL("coral"),
CORNFLOWERBLUE("cornflowerblue"),
CORNSILK("cornsilk"),
CRIMSON("crimson"),
CYAN("cyan"),
DARKBLUE("darkblue"),
DARKCYAN("darkcyan"),
DARKGOLDENROD("darkgoldenrod"),
DARKGRAY("darkgray"),
DARKGREEN("darkgreen"),
DARKKHAKI("darkkhaki"),
DARKMAGENTA("darkmagenta"),
DARKOLIVEGREEN("darkolivegreen"),
DARKORANGE("darkorange"),
DARKORCHID("darkorchid"),
DARKRED("darkred"),
DARKSALMON("darksalmon"),
DARKSEAGREEN("darkseagreen"),
DARKSLATEBLUE("darkslateblue"),
DARKSLATEGRAY("darkslategray"),
DARKTURQUOISE("darkturquoise"),
DARKVIOLET("darkviolet"),
DEEPPINK("deeppink"),
DEEPSKYBLUE("deepskyblue"),
DIMGRAY("dimgray"),
DODGERBLUE("dodgerblue"),
FIREBRICK("firebrick"),
FLORALWHITE("floralwhite"),
FORESTGREEN("forestgreen"),
FUCHSIA("fuchsia"),
GAINSBORO("gainsboro"),
GHOSTWHITE("ghostwhite"),
GOLD("gold"),
GOLDENROD("goldenrod"),
GRAY("gray"),
GREEN("green"),
GREENYELLOW("greenyellow"),
HONEYDEW("honeydew"),
HOTPINK("hotpink"),
INDIANRED("indianred"),
INDIGO("indigo"),
IVORY("ivory"),
KHAKI("khaki"),
LAVENDER("lavender"),
LAVENDERBLUSH("lavenderblush"),
LAWNGREEN("lawngreen"),
LEMONCHIFFON("lemonchiffon"),
LIGHTBLUE("lightblue"),
LIGHTCORAL("lightcoral"),
LIGHTCYAN("lightcyan"),
LIGHTGOLDENRODYE("lightgoldenrodye"),
LIGHTGREEN("lightgreen"),
LIGHTGREY("lightgrey"),
LIGHTPINK("lightpink"),
LIGHTSALMON("lightsalmon"),
LIGHTSEAGREEN("lightseagreen"),
LIGHTSKYBLUE("lightskyblue"),
LIGHTSLATEGRAY("lightslategray"),
LIGHTSTEELBLUE("lightsteelblue"),
LIGHTYELLOW("lightyellow"),
LIME("lime"),
LIMEGREEN("limegreen"),
LINEN("linen"),
MAGENTA("magenta"),
MAROON("maroon"),
MEDIUMAQUAMARINE("mediumaquamarine"),
MEDIUMBLUE("mediumblue"),
MEDIUMORCHID("mediumorchid"),
MEDIUMPURPLE("mediumpurple"),
MEDIUMSEAGREEN("mediumseagreen"),
MEDIUMSLATEBLUE("mediumslateblue"),
MEDIUMSPRINGGREE("mediumspringgree"),
MEDIUMTURQUOISE("mediumturquoise"),
MEDIUMVIOLETRED("mediumvioletred"),
MIDNIGHTBLUE("midnightblue"),
MINTCREAM("mintcream"),
MISTYROSE("mistyrose"),
MOCCASIN("moccasin"),
NAVAJOWHITE("navajowhite"),
NAVY("navy"),
NAVYBLUE("navyblue"),
OLDLACE("oldlace"),
OLIVE("olive"),
OLIVEDRAB("olivedrab"),
ORANGE("orange"),
ORANGERED("orangered"),
ORCHID("orchid"),
PALEGOLDENROD("palegoldenrod"),
PALEGREEN("palegreen"),
PALETURQUOISE("paleturquoise"),
PALEVIOLETRED("palevioletred"),
PAPAYAWHIP("papayawhip"),
PEACHPUFF("peachpuff"),
PERU("peru"),
PINK("pink"),
PLUM("plum"),
POWDERBLUE("powderblue"),
PURPLE("purple"),
RED("red"),
ROSYBROWN("rosybrown"),
ROYALBLUE("royalblue"),
SADDLEBROWN("saddlebrown"),
SALMON("salmon"),
SANDYBROWN("sandybrown"),
SEAGREEN("seagreen"),
SEASHELL("seashell"),
SIENNA("sienna"),
SILVER("silver"),
SKYBLUE("skyblue"),
SLATEBLUE("slateblue"),
SLATEGRAY("slategray"),
SNOW("snow"),
SPRINGGREEN("springgreen"),
STEELBLUE("steelblue"),
TAN("tan"),
TEAL("teal"),
THISTLE("thistle"),
TOMATO("tomato"),
TURQUOISE("turquoise"),
VIOLET("violet"),
WHEAT("wheat"),
WHITE("white"),
WHITESMOKE("whitesmoke"),
YELLOW("yellow"),
YELLOWGREEN("yellowgreen");
private final String color;
private GnuplotColor(final String color) {
this.color = color;
}
public String getColor() {
return color;
}
}

43
recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java Normal file
View File

@@ -0,0 +1,43 @@
package org.lucares.recommind.logs;
import java.util.Collection;
public class GnuplotFileGenerator {
public String generate(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final StringBuilder result = new StringBuilder();
appendfln(result, "set terminal %s size %d,%d", settings.getTerminal(), settings.getWidth(),
settings.getHeight());
appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator());
appendfln(result, "set timefmt '%s'", settings.getTimefmt());
appendfln(result, "set xdata time");
appendfln(result, "set format x \"%s\"", settings.getFormatX());
appendfln(result, "set xlabel \"%s\"", settings.getXlabel());
appendfln(result, "set xtics rotate by %d", settings.getRotateXAxisLabel());
appendfln(result, "set ylabel \"%s\"", settings.getYlabel());
appendfln(result, "set output \"%s\"", settings.getOutput().getAbsolutePath());
appendf(result, "plot ");
for (final DataSeries dataSerie : dataSeries) {
appendfln(result, "'%s' using 1:2 title '%s' with points, \\", dataSerie.getDataFile(),
dataSerie.getTitle());
}
return result.toString();
}
private void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(format + "\n", args));
}
private void appendf(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(format, args));
}
}

115
recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java Normal file
View File

@@ -0,0 +1,115 @@
package org.lucares.recommind.logs;
import java.io.File;
public class GnuplotSettings {
private String terminal = "png";
private int height = 1200;
private int width = 1600;
private String timefmt = "%Y-%m-%dT%H:%M:%S";
// set format x "%m-%d\n%H:%M"
private String formatX = "%Y-%m-%d %H:%M:%S";
// set datafile separator ","
private String datafileSeparator = ",";
// set xlabel "Time"
private String xlabel = "Time";
// set ylabel "Traffic"
private String ylabel = "Duration in ms";
// set output "datausage.png"
private File output = new File("/tmp/out.png");
// set xtics rotate by 80
private int rotateXAxisLabel = -80;
public GnuplotSettings(final File output) {
this.output = output;
}
public int getRotateXAxisLabel() {
return rotateXAxisLabel;
}
public void setRotateXAxisLabel(final int rotateXAxisLabel) {
this.rotateXAxisLabel = rotateXAxisLabel;
}
public String getTerminal() {
return terminal;
}
public void setTerminal(final String terminal) {
this.terminal = terminal;
}
public int getHeight() {
return height;
}
public void setHeight(final int height) {
this.height = height;
}
public int getWidth() {
return width;
}
public void setWidth(final int width) {
this.width = width;
}
public String getTimefmt() {
return timefmt;
}
public void setTimefmt(final String timefmt) {
this.timefmt = timefmt;
}
public String getFormatX() {
return formatX;
}
public void setFormatX(final String formatX) {
this.formatX = formatX;
}
public String getDatafileSeparator() {
return datafileSeparator;
}
public void setDatafileSeparator(final String datafileSeparator) {
this.datafileSeparator = datafileSeparator;
}
public String getXlabel() {
return xlabel;
}
public void setXlabel(final String xlabel) {
this.xlabel = xlabel;
}
public String getYlabel() {
return ylabel;
}
public void setYlabel(final String ylabel) {
this.ylabel = ylabel;
}
public File getOutput() {
return output;
}
public void setOutput(final File output) {
this.output = output;
}
// plot 'sample.txt' using 1:2 title 'Bytes' with linespoints 2
}

73
recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java Normal file
View File

@@ -0,0 +1,73 @@
package org.lucares.recommind.logs;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.stream.Stream;
import org.lucares.performance.db.Entry;
import org.lucares.performance.db.FileUtils;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.tanga.svak.StreamSvWriter;
import org.lucares.tanga.svak.StreamSvWriterSettings;
import org.lucares.tanga.svak.StreamSvWriterSettings.Escaping;
import org.lucares.tanga.svak.StreamSvWriterSettings.FieldQuoting;
public class Plotter {
public static void main(final String[] args) throws Exception {
final Path dataDirectory = Paths.get(args[0]);
final Path outputDirectory = Paths.get(args[1]);
final String query = args[2];
final Path tmpBaseDir = Paths.get(args[0], "tmp");
Files.createDirectories(tmpBaseDir);
Files.createDirectories(outputDirectory);
final Path tmpDirectory = Files.createTempDirectory(tmpBaseDir, "gnuplot");
try {
final Collection<DataSeries> dataSeries = new ArrayList<>();
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final Stream<Entry> entries = db.get(query);
final File dataFile = File.createTempFile("data", ".dat", tmpDirectory.toFile());
final DataSeries dataSerie = new DataSeries(dataFile, query);
toCsv(entries, dataFile);
dataSeries.add(dataSerie);
}
final File outputFile = File.createTempFile("out", ".png", outputDirectory.toFile());
final Gnuplot gnuplot = new Gnuplot(tmpDirectory);
final GnuplotSettings settings = new GnuplotSettings(outputFile);
gnuplot.plot(settings, dataSeries);
} finally {
FileUtils.delete(tmpDirectory);
}
}
private static void toCsv(final Stream<Entry> entries, final File dataFile) throws IOException {
final StreamSvWriterSettings svSettings = new StreamSvWriterSettings(",", "'");
svSettings.setFieldQuoting(FieldQuoting.IF_NEEDED);
svSettings.setEscaping(Escaping.NONE);
try (FileOutputStream output = new FileOutputStream(dataFile);
StreamSvWriter writer = new StreamSvWriter(output, svSettings)) {
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
final Entry entry = it.next();
final String value = String.valueOf(entry.getValue());
final String date = entry.getDate().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
writer.writeLine(date, value);
}
}
}
}