render plot with a single dataseriew

performanceDb/src/main/java/org/lucares/performance/db/Fields.java

@@ -1,23 +1,11 @@
 package org.lucares.performance.db;
 
 class Fields {
-	static final String TAG_PREFIX = "__tag_";
+	static final String PREFIX_INTERNAL_TAG = "__internal_";
 
-	static final String DATE_OFFSET = "__date_offset__";
+	static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset";
 
-	static String prefixedKey(final String key) {
+	static boolean isInternalField(final String key) {
-		return TAG_PREFIX + key;
+		return key.startsWith(PREFIX_INTERNAL_TAG);
 	}
-
-	static boolean isPrefixedKey(final String key) {
-		return key.startsWith(TAG_PREFIX);
-	}
-
-	static String stripPrefix(final String key) {
-		if (!isPrefixedKey(key)) {
-			throw new IllegalArgumentException(key + " is not prefixed by " + TAG_PREFIX);
-		}
-		return key.substring(TAG_PREFIX.length());
-	}
-
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java

@@ -31,6 +31,9 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
 			if (reader == null) {
 				nextFile();
 			}
+			if (reader == null) {
+				return null;
+			}
 			final Optional<Entry> optionalEntry = reader.readEntry(currentPdbFile.getTags());
 
 			return optionalEntry.orElseGet(() -> {

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -115,7 +115,22 @@ public class PerformanceDb implements AutoCloseable {
 	public Stream<Entry> get(final Tags tags) {
 
 		final List<PdbFile> pdbFiles = tagsToFile.getFilesMatchingTags(tags);
+		return toStream(pdbFiles);
+	}
 
+	/**
+	 * Return the entries as an unbound, ordered and non-parallel stream.
+	 *
+	 * @param query
+	 * @return {@link Stream} unbound, ordered and non-parallel
+	 */
+	public Stream<Entry> get(final String query) {
+
+		final List<PdbFile> pdbFiles = tagsToFile.getFilesForQuery(query);
+		return toStream(pdbFiles);
+	}
+
+	private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
 		final Iterator<Entry> iterator = new PdbFileIterator(pdbFiles);
 
 		final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);

performanceDb/src/main/java/org/lucares/performance/db/Query.java

@@ -6,7 +6,7 @@ final class Query {
 
 		for (final String key : tags.getKeys()) {
 			tags.getValue(key).ifPresent(value -> {
-				result.append(Fields.prefixedKey(key));
+				result.append(key);
 				result.append("=");
 				result.append(value);
 				result.append(" ");

performanceDb/src/main/java/org/lucares/performance/db/Tags.java

@@ -21,6 +21,15 @@ public class Tags {
 
 	private Tags(final Map<String, String> tags) {
 		this.tags.putAll(tags);
+		ensureNoInternalFields();
+	}
+
+	private void ensureNoInternalFields() {
+		tags.keySet().forEach(key -> {
+			if (Fields.isInternalField(key)) {
+				throw new IllegalArgumentException(key + " is an internal field. Choose another prefix.");
+			}
+		});
 	}
 
 	public static Tags create() {

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -48,8 +48,12 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 	}
 
 	List<PdbFile> getFilesMatchingTags(final Tags tags) {
-		final List<PdbFile> result = new ArrayList<>();
 		final String query = Query.createQuery(tags);
+		return getFilesForQuery(query);
+	}
+
+	List<PdbFile> getFilesForQuery(final String query) {
+		final List<PdbFile> result = new ArrayList<>();
 		try {
 			final List<Document> searchResult = db.search(query);
 
@@ -81,9 +85,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 
 		for (final String key : document.getProperties().keySet()) {
 
-			if (Fields.isPrefixedKey(key)) {
+			if (!Fields.isInternalField(key)) {
 				final String value = document.getPropertyString(key);
-				tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value);
+				tagsOfFile = tagsOfFile.copyAdd(key, value);
 			}
 		}
 		return tagsOfFile;
@@ -130,7 +134,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 		ensureFieldsExist(tags);
 
 		tags.forEach((key, value) -> {
-			db.setProperty(file, Fields.prefixedKey(key), value);
+			db.setProperty(file, key, value);
 		});
 
 		db.setProperty(file, Fields.DATE_OFFSET, day.serialize());
@@ -145,11 +149,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 
 		tags.forEach((key, value) -> {
 
-			final String prefixedKey = Fields.prefixedKey(key);
+			final Field field = fieldsMap.get(key);
-
-			final Field field = fieldsMap.get(prefixedKey);
 			if (field == null) {
-				db.createField(prefixedKey, FieldType.STRING);
+				db.createField(key, FieldType.STRING);
 			}
 		});
 	}

recommind-logs/build.gradle

@@ -2,5 +2,6 @@
 dependencies {
 	compile project(':performanceDb')
 	compile "io.thekraken:grok:0.1.5"
+	compile 'org.lucares:svak:1.0'
 }
 

recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java

@@ -0,0 +1,45 @@
+package org.lucares.recommind.logs;
+
+import java.io.File;
+
+public class DataSeries {
+	private final File dataFile;
+
+	private final String title;
+
+	private final GnuplotColor color;
+
+	private final Integer pointType;
+
+	public DataSeries(final File dataFile, final String title) {
+		super();
+		this.dataFile = dataFile;
+		this.title = title;
+		this.color = null;
+		this.pointType = null;
+	}
+
+	public DataSeries(final File dataFile, final String title, final GnuplotColor color, final Integer pointType) {
+		super();
+		this.dataFile = dataFile;
+		this.title = title;
+		this.color = color;
+		this.pointType = pointType;
+	}
+
+	public GnuplotColor getColor() {
+		return color;
+	}
+
+	public Integer getPointType() {
+		return pointType;
+	}
+
+	public File getDataFile() {
+		return dataFile;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java

@@ -0,0 +1,36 @@
+package org.lucares.recommind.logs;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Collection;
+
+import com.google.common.io.Files;
+
+public class Gnuplot {
+
+	private final Path tmpDirectory;
+
+	public Gnuplot(final Path tmpDirectory) {
+		this.tmpDirectory = tmpDirectory;
+	}
+
+	public void plot(final GnuplotSettings settings, final Collection<DataSeries> dataSeries)
+			throws IOException, InterruptedException {
+
+		final GnuplotFileGenerator generator = new GnuplotFileGenerator();
+
+		final String gnuplotFileContent = generator.generate(settings, dataSeries);
+		System.out.println(gnuplotFileContent);
+
+		final File gnuplotFile = File.createTempFile("gnuplot", ".dem", tmpDirectory.toFile());
+		Files.write(gnuplotFileContent, gnuplotFile, StandardCharsets.UTF_8);
+
+		final ProcessBuilder processBuilder = new ProcessBuilder("gnuplot", gnuplotFile.getAbsolutePath());
+		processBuilder.inheritIO();
+		final Process start = processBuilder.start();
+		start.waitFor();
+
+	}
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java

@@ -0,0 +1,296 @@
+package org.lucares.recommind.logs;
+
+public enum GnuplotColor {
+
+	ALICEBLUE("aliceblue"),
+
+	ANTIQUEWHITE("antiquewhite"),
+
+	AQUA("aqua"),
+
+	AQUAMARINE("aquamarine"),
+
+	AZURE("azure"),
+
+	BEIGE("beige"),
+
+	BISQUE("bisque"),
+
+	BLACK("black"),
+
+	BLANCHEDALMOND("blanchedalmond"),
+
+	BLUE("blue"),
+
+	BLUEVIOLET("blueviolet"),
+
+	BROWN("brown"),
+
+	BURLYWOOD("burlywood"),
+
+	CADETBLUE("cadetblue"),
+
+	CHARTREUSE("chartreuse"),
+
+	CHOCOLATE("chocolate"),
+
+	CORAL("coral"),
+
+	CORNFLOWERBLUE("cornflowerblue"),
+
+	CORNSILK("cornsilk"),
+
+	CRIMSON("crimson"),
+
+	CYAN("cyan"),
+
+	DARKBLUE("darkblue"),
+
+	DARKCYAN("darkcyan"),
+
+	DARKGOLDENROD("darkgoldenrod"),
+
+	DARKGRAY("darkgray"),
+
+	DARKGREEN("darkgreen"),
+
+	DARKKHAKI("darkkhaki"),
+
+	DARKMAGENTA("darkmagenta"),
+
+	DARKOLIVEGREEN("darkolivegreen"),
+
+	DARKORANGE("darkorange"),
+
+	DARKORCHID("darkorchid"),
+
+	DARKRED("darkred"),
+
+	DARKSALMON("darksalmon"),
+
+	DARKSEAGREEN("darkseagreen"),
+
+	DARKSLATEBLUE("darkslateblue"),
+
+	DARKSLATEGRAY("darkslategray"),
+
+	DARKTURQUOISE("darkturquoise"),
+
+	DARKVIOLET("darkviolet"),
+
+	DEEPPINK("deeppink"),
+
+	DEEPSKYBLUE("deepskyblue"),
+
+	DIMGRAY("dimgray"),
+
+	DODGERBLUE("dodgerblue"),
+
+	FIREBRICK("firebrick"),
+
+	FLORALWHITE("floralwhite"),
+
+	FORESTGREEN("forestgreen"),
+
+	FUCHSIA("fuchsia"),
+
+	GAINSBORO("gainsboro"),
+
+	GHOSTWHITE("ghostwhite"),
+
+	GOLD("gold"),
+
+	GOLDENROD("goldenrod"),
+
+	GRAY("gray"),
+
+	GREEN("green"),
+
+	GREENYELLOW("greenyellow"),
+
+	HONEYDEW("honeydew"),
+
+	HOTPINK("hotpink"),
+
+	INDIANRED("indianred"),
+
+	INDIGO("indigo"),
+
+	IVORY("ivory"),
+
+	KHAKI("khaki"),
+
+	LAVENDER("lavender"),
+
+	LAVENDERBLUSH("lavenderblush"),
+
+	LAWNGREEN("lawngreen"),
+
+	LEMONCHIFFON("lemonchiffon"),
+
+	LIGHTBLUE("lightblue"),
+
+	LIGHTCORAL("lightcoral"),
+
+	LIGHTCYAN("lightcyan"),
+
+	LIGHTGOLDENRODYE("lightgoldenrodye"),
+
+	LIGHTGREEN("lightgreen"),
+
+	LIGHTGREY("lightgrey"),
+
+	LIGHTPINK("lightpink"),
+
+	LIGHTSALMON("lightsalmon"),
+
+	LIGHTSEAGREEN("lightseagreen"),
+
+	LIGHTSKYBLUE("lightskyblue"),
+
+	LIGHTSLATEGRAY("lightslategray"),
+
+	LIGHTSTEELBLUE("lightsteelblue"),
+
+	LIGHTYELLOW("lightyellow"),
+
+	LIME("lime"),
+
+	LIMEGREEN("limegreen"),
+
+	LINEN("linen"),
+
+	MAGENTA("magenta"),
+
+	MAROON("maroon"),
+
+	MEDIUMAQUAMARINE("mediumaquamarine"),
+
+	MEDIUMBLUE("mediumblue"),
+
+	MEDIUMORCHID("mediumorchid"),
+
+	MEDIUMPURPLE("mediumpurple"),
+
+	MEDIUMSEAGREEN("mediumseagreen"),
+
+	MEDIUMSLATEBLUE("mediumslateblue"),
+
+	MEDIUMSPRINGGREE("mediumspringgree"),
+
+	MEDIUMTURQUOISE("mediumturquoise"),
+
+	MEDIUMVIOLETRED("mediumvioletred"),
+
+	MIDNIGHTBLUE("midnightblue"),
+
+	MINTCREAM("mintcream"),
+
+	MISTYROSE("mistyrose"),
+
+	MOCCASIN("moccasin"),
+
+	NAVAJOWHITE("navajowhite"),
+
+	NAVY("navy"),
+
+	NAVYBLUE("navyblue"),
+
+	OLDLACE("oldlace"),
+
+	OLIVE("olive"),
+
+	OLIVEDRAB("olivedrab"),
+
+	ORANGE("orange"),
+
+	ORANGERED("orangered"),
+
+	ORCHID("orchid"),
+
+	PALEGOLDENROD("palegoldenrod"),
+
+	PALEGREEN("palegreen"),
+
+	PALETURQUOISE("paleturquoise"),
+
+	PALEVIOLETRED("palevioletred"),
+
+	PAPAYAWHIP("papayawhip"),
+
+	PEACHPUFF("peachpuff"),
+
+	PERU("peru"),
+
+	PINK("pink"),
+
+	PLUM("plum"),
+
+	POWDERBLUE("powderblue"),
+
+	PURPLE("purple"),
+
+	RED("red"),
+
+	ROSYBROWN("rosybrown"),
+
+	ROYALBLUE("royalblue"),
+
+	SADDLEBROWN("saddlebrown"),
+
+	SALMON("salmon"),
+
+	SANDYBROWN("sandybrown"),
+
+	SEAGREEN("seagreen"),
+
+	SEASHELL("seashell"),
+
+	SIENNA("sienna"),
+
+	SILVER("silver"),
+
+	SKYBLUE("skyblue"),
+
+	SLATEBLUE("slateblue"),
+
+	SLATEGRAY("slategray"),
+
+	SNOW("snow"),
+
+	SPRINGGREEN("springgreen"),
+
+	STEELBLUE("steelblue"),
+
+	TAN("tan"),
+
+	TEAL("teal"),
+
+	THISTLE("thistle"),
+
+	TOMATO("tomato"),
+
+	TURQUOISE("turquoise"),
+
+	VIOLET("violet"),
+
+	WHEAT("wheat"),
+
+	WHITE("white"),
+
+	WHITESMOKE("whitesmoke"),
+
+	YELLOW("yellow"),
+
+	YELLOWGREEN("yellowgreen");
+
+	private final String color;
+
+	private GnuplotColor(final String color) {
+		this.color = color;
+	}
+
+	public String getColor() {
+		return color;
+	}
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -0,0 +1,43 @@
+package org.lucares.recommind.logs;
+
+import java.util.Collection;
+
+public class GnuplotFileGenerator {
+
+	public String generate(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
+
+		final StringBuilder result = new StringBuilder();
+
+		appendfln(result, "set terminal %s size %d,%d", settings.getTerminal(), settings.getWidth(),
+				settings.getHeight());
+
+		appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator());
+		appendfln(result, "set timefmt '%s'", settings.getTimefmt());
+
+		appendfln(result, "set xdata time");
+		appendfln(result, "set format x \"%s\"", settings.getFormatX());
+		appendfln(result, "set xlabel \"%s\"", settings.getXlabel());
+		appendfln(result, "set xtics rotate by %d", settings.getRotateXAxisLabel());
+
+		appendfln(result, "set ylabel \"%s\"", settings.getYlabel());
+
+		appendfln(result, "set output \"%s\"", settings.getOutput().getAbsolutePath());
+		appendf(result, "plot ");
+
+		for (final DataSeries dataSerie : dataSeries) {
+			appendfln(result, "'%s' using 1:2 title '%s' with points, \\", dataSerie.getDataFile(),
+					dataSerie.getTitle());
+		}
+
+		return result.toString();
+	}
+
+	private void appendfln(final StringBuilder builder, final String format, final Object... args) {
+		builder.append(String.format(format + "\n", args));
+	}
+
+	private void appendf(final StringBuilder builder, final String format, final Object... args) {
+		builder.append(String.format(format, args));
+	}
+
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java

@@ -0,0 +1,115 @@
+package org.lucares.recommind.logs;
+
+import java.io.File;
+
+public class GnuplotSettings {
+	private String terminal = "png";
+	private int height = 1200;
+	private int width = 1600;
+	private String timefmt = "%Y-%m-%dT%H:%M:%S";
+
+	// set format x "%m-%d\n%H:%M"
+	private String formatX = "%Y-%m-%d %H:%M:%S";
+
+	// set datafile separator ","
+	private String datafileSeparator = ",";
+
+	// set xlabel "Time"
+	private String xlabel = "Time";
+
+	// set ylabel "Traffic"
+	private String ylabel = "Duration in ms";
+
+	// set output "datausage.png"
+	private File output = new File("/tmp/out.png");
+
+	// set xtics rotate by 80
+	private int rotateXAxisLabel = -80;
+
+	public GnuplotSettings(final File output) {
+		this.output = output;
+	}
+
+	public int getRotateXAxisLabel() {
+		return rotateXAxisLabel;
+	}
+
+	public void setRotateXAxisLabel(final int rotateXAxisLabel) {
+		this.rotateXAxisLabel = rotateXAxisLabel;
+	}
+
+	public String getTerminal() {
+		return terminal;
+	}
+
+	public void setTerminal(final String terminal) {
+		this.terminal = terminal;
+	}
+
+	public int getHeight() {
+		return height;
+	}
+
+	public void setHeight(final int height) {
+		this.height = height;
+	}
+
+	public int getWidth() {
+		return width;
+	}
+
+	public void setWidth(final int width) {
+		this.width = width;
+	}
+
+	public String getTimefmt() {
+		return timefmt;
+	}
+
+	public void setTimefmt(final String timefmt) {
+		this.timefmt = timefmt;
+	}
+
+	public String getFormatX() {
+		return formatX;
+	}
+
+	public void setFormatX(final String formatX) {
+		this.formatX = formatX;
+	}
+
+	public String getDatafileSeparator() {
+		return datafileSeparator;
+	}
+
+	public void setDatafileSeparator(final String datafileSeparator) {
+		this.datafileSeparator = datafileSeparator;
+	}
+
+	public String getXlabel() {
+		return xlabel;
+	}
+
+	public void setXlabel(final String xlabel) {
+		this.xlabel = xlabel;
+	}
+
+	public String getYlabel() {
+		return ylabel;
+	}
+
+	public void setYlabel(final String ylabel) {
+		this.ylabel = ylabel;
+	}
+
+	public File getOutput() {
+		return output;
+	}
+
+	public void setOutput(final File output) {
+		this.output = output;
+	}
+
+	// plot 'sample.txt' using 1:2 title 'Bytes' with linespoints 2
+
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java

@@ -0,0 +1,73 @@
+package org.lucares.recommind.logs;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.stream.Stream;
+
+import org.lucares.performance.db.Entry;
+import org.lucares.performance.db.FileUtils;
+import org.lucares.performance.db.PerformanceDb;
+import org.lucares.tanga.svak.StreamSvWriter;
+import org.lucares.tanga.svak.StreamSvWriterSettings;
+import org.lucares.tanga.svak.StreamSvWriterSettings.Escaping;
+import org.lucares.tanga.svak.StreamSvWriterSettings.FieldQuoting;
+
+public class Plotter {
+	public static void main(final String[] args) throws Exception {
+		final Path dataDirectory = Paths.get(args[0]);
+		final Path outputDirectory = Paths.get(args[1]);
+		final String query = args[2];
+		final Path tmpBaseDir = Paths.get(args[0], "tmp");
+		Files.createDirectories(tmpBaseDir);
+		Files.createDirectories(outputDirectory);
+		final Path tmpDirectory = Files.createTempDirectory(tmpBaseDir, "gnuplot");
+		try {
+
+			final Collection<DataSeries> dataSeries = new ArrayList<>();
+
+			try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+				final Stream<Entry> entries = db.get(query);
+
+				final File dataFile = File.createTempFile("data", ".dat", tmpDirectory.toFile());
+				final DataSeries dataSerie = new DataSeries(dataFile, query);
+				toCsv(entries, dataFile);
+
+				dataSeries.add(dataSerie);
+			}
+
+			final File outputFile = File.createTempFile("out", ".png", outputDirectory.toFile());
+			final Gnuplot gnuplot = new Gnuplot(tmpDirectory);
+			final GnuplotSettings settings = new GnuplotSettings(outputFile);
+			gnuplot.plot(settings, dataSeries);
+		} finally {
+			FileUtils.delete(tmpDirectory);
+		}
+	}
+
+	private static void toCsv(final Stream<Entry> entries, final File dataFile) throws IOException {
+
+		final StreamSvWriterSettings svSettings = new StreamSvWriterSettings(",", "'");
+		svSettings.setFieldQuoting(FieldQuoting.IF_NEEDED);
+		svSettings.setEscaping(Escaping.NONE);
+		try (FileOutputStream output = new FileOutputStream(dataFile);
+				StreamSvWriter writer = new StreamSvWriter(output, svSettings)) {
+
+			final Iterator<Entry> it = entries.iterator();
+			while (it.hasNext()) {
+				final Entry entry = it.next();
+
+				final String value = String.valueOf(entry.getValue());
+				final String date = entry.getDate().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
+				writer.writeLine(date, value);
+			}
+		}
+	}
+}