From e936df6f7eec88d4c780039b174af9bc73b4c2ce Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 10 Dec 2016 18:50:29 +0100 Subject: [PATCH] render plot with a single dataseriew --- .../org/lucares/performance/db/Fields.java | 20 +- .../performance/db/PdbFileIterator.java | 3 + .../lucares/performance/db/PerformanceDb.java | 15 + .../org/lucares/performance/db/Query.java | 2 +- .../java/org/lucares/performance/db/Tags.java | 9 + .../lucares/performance/db/TagsToFile.java | 18 +- recommind-logs/build.gradle | 1 + .../lucares/recommind/logs/DataSeries.java | 45 +++ .../org/lucares/recommind/logs/Gnuplot.java | 36 +++ .../lucares/recommind/logs/GnuplotColor.java | 296 ++++++++++++++++++ .../recommind/logs/GnuplotFileGenerator.java | 43 +++ .../recommind/logs/GnuplotSettings.java | 115 +++++++ .../org/lucares/recommind/logs/Plotter.java | 73 +++++ 13 files changed, 651 insertions(+), 25 deletions(-) create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Fields.java b/performanceDb/src/main/java/org/lucares/performance/db/Fields.java index 224331b..2019c20 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Fields.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Fields.java @@ -1,23 +1,11 @@ package org.lucares.performance.db; class Fields { - static final String TAG_PREFIX = "__tag_"; + static final String PREFIX_INTERNAL_TAG = "__internal_"; - static final String DATE_OFFSET = "__date_offset__"; + static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset"; - static String prefixedKey(final String key) { - return TAG_PREFIX + key; + static boolean isInternalField(final String key) { + return key.startsWith(PREFIX_INTERNAL_TAG); } - - static boolean isPrefixedKey(final String key) { - return key.startsWith(TAG_PREFIX); - } - - static String stripPrefix(final String key) { - if (!isPrefixedKey(key)) { - throw new IllegalArgumentException(key + " is not prefixed by " + TAG_PREFIX); - } - return key.substring(TAG_PREFIX.length()); - } - } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index 83135da..e59c174 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -31,6 +31,9 @@ public class PdbFileIterator implements Iterator, AutoCloseable { if (reader == null) { nextFile(); } + if (reader == null) { + return null; + } final Optional optionalEntry = reader.readEntry(currentPdbFile.getTags()); return optionalEntry.orElseGet(() -> { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index 526b250..d7ae5b5 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -115,7 +115,22 @@ public class PerformanceDb implements AutoCloseable { public Stream get(final Tags tags) { final List pdbFiles = tagsToFile.getFilesMatchingTags(tags); + return toStream(pdbFiles); + } + /** + * Return the entries as an unbound, ordered and non-parallel stream. + * + * @param query + * @return {@link Stream} unbound, ordered and non-parallel + */ + public Stream get(final String query) { + + final List pdbFiles = tagsToFile.getFilesForQuery(query); + return toStream(pdbFiles); + } + + private Stream toStream(final List pdbFiles) { final Iterator iterator = new PdbFileIterator(pdbFiles); final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Query.java b/performanceDb/src/main/java/org/lucares/performance/db/Query.java index e099b4c..c715cb5 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Query.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Query.java @@ -6,7 +6,7 @@ final class Query { for (final String key : tags.getKeys()) { tags.getValue(key).ifPresent(value -> { - result.append(Fields.prefixedKey(key)); + result.append(key); result.append("="); result.append(value); result.append(" "); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Tags.java b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java index 8afa573..a630263 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Tags.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java @@ -21,6 +21,15 @@ public class Tags { private Tags(final Map tags) { this.tags.putAll(tags); + ensureNoInternalFields(); + } + + private void ensureNoInternalFields() { + tags.keySet().forEach(key -> { + if (Fields.isInternalField(key)) { + throw new IllegalArgumentException(key + " is an internal field. Choose another prefix."); + } + }); } public static Tags create() { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index db5694d..5e932c7 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -48,8 +48,12 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { } List getFilesMatchingTags(final Tags tags) { - final List result = new ArrayList<>(); final String query = Query.createQuery(tags); + return getFilesForQuery(query); + } + + List getFilesForQuery(final String query) { + final List result = new ArrayList<>(); try { final List searchResult = db.search(query); @@ -81,9 +85,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { for (final String key : document.getProperties().keySet()) { - if (Fields.isPrefixedKey(key)) { + if (!Fields.isInternalField(key)) { final String value = document.getPropertyString(key); - tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value); + tagsOfFile = tagsOfFile.copyAdd(key, value); } } return tagsOfFile; @@ -130,7 +134,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { ensureFieldsExist(tags); tags.forEach((key, value) -> { - db.setProperty(file, Fields.prefixedKey(key), value); + db.setProperty(file, key, value); }); db.setProperty(file, Fields.DATE_OFFSET, day.serialize()); @@ -145,11 +149,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { tags.forEach((key, value) -> { - final String prefixedKey = Fields.prefixedKey(key); - - final Field field = fieldsMap.get(prefixedKey); + final Field field = fieldsMap.get(key); if (field == null) { - db.createField(prefixedKey, FieldType.STRING); + db.createField(key, FieldType.STRING); } }); } diff --git a/recommind-logs/build.gradle b/recommind-logs/build.gradle index d7a74dd..ee8b9c8 100644 --- a/recommind-logs/build.gradle +++ b/recommind-logs/build.gradle @@ -2,5 +2,6 @@ dependencies { compile project(':performanceDb') compile "io.thekraken:grok:0.1.5" + compile 'org.lucares:svak:1.0' } diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java new file mode 100644 index 0000000..fe183b7 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/DataSeries.java @@ -0,0 +1,45 @@ +package org.lucares.recommind.logs; + +import java.io.File; + +public class DataSeries { + private final File dataFile; + + private final String title; + + private final GnuplotColor color; + + private final Integer pointType; + + public DataSeries(final File dataFile, final String title) { + super(); + this.dataFile = dataFile; + this.title = title; + this.color = null; + this.pointType = null; + } + + public DataSeries(final File dataFile, final String title, final GnuplotColor color, final Integer pointType) { + super(); + this.dataFile = dataFile; + this.title = title; + this.color = color; + this.pointType = pointType; + } + + public GnuplotColor getColor() { + return color; + } + + public Integer getPointType() { + return pointType; + } + + public File getDataFile() { + return dataFile; + } + + public String getTitle() { + return title; + } +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java new file mode 100644 index 0000000..540294c --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java @@ -0,0 +1,36 @@ +package org.lucares.recommind.logs; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collection; + +import com.google.common.io.Files; + +public class Gnuplot { + + private final Path tmpDirectory; + + public Gnuplot(final Path tmpDirectory) { + this.tmpDirectory = tmpDirectory; + } + + public void plot(final GnuplotSettings settings, final Collection dataSeries) + throws IOException, InterruptedException { + + final GnuplotFileGenerator generator = new GnuplotFileGenerator(); + + final String gnuplotFileContent = generator.generate(settings, dataSeries); + System.out.println(gnuplotFileContent); + + final File gnuplotFile = File.createTempFile("gnuplot", ".dem", tmpDirectory.toFile()); + Files.write(gnuplotFileContent, gnuplotFile, StandardCharsets.UTF_8); + + final ProcessBuilder processBuilder = new ProcessBuilder("gnuplot", gnuplotFile.getAbsolutePath()); + processBuilder.inheritIO(); + final Process start = processBuilder.start(); + start.waitFor(); + + } +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java new file mode 100644 index 0000000..0fc1480 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotColor.java @@ -0,0 +1,296 @@ +package org.lucares.recommind.logs; + +public enum GnuplotColor { + + ALICEBLUE("aliceblue"), + + ANTIQUEWHITE("antiquewhite"), + + AQUA("aqua"), + + AQUAMARINE("aquamarine"), + + AZURE("azure"), + + BEIGE("beige"), + + BISQUE("bisque"), + + BLACK("black"), + + BLANCHEDALMOND("blanchedalmond"), + + BLUE("blue"), + + BLUEVIOLET("blueviolet"), + + BROWN("brown"), + + BURLYWOOD("burlywood"), + + CADETBLUE("cadetblue"), + + CHARTREUSE("chartreuse"), + + CHOCOLATE("chocolate"), + + CORAL("coral"), + + CORNFLOWERBLUE("cornflowerblue"), + + CORNSILK("cornsilk"), + + CRIMSON("crimson"), + + CYAN("cyan"), + + DARKBLUE("darkblue"), + + DARKCYAN("darkcyan"), + + DARKGOLDENROD("darkgoldenrod"), + + DARKGRAY("darkgray"), + + DARKGREEN("darkgreen"), + + DARKKHAKI("darkkhaki"), + + DARKMAGENTA("darkmagenta"), + + DARKOLIVEGREEN("darkolivegreen"), + + DARKORANGE("darkorange"), + + DARKORCHID("darkorchid"), + + DARKRED("darkred"), + + DARKSALMON("darksalmon"), + + DARKSEAGREEN("darkseagreen"), + + DARKSLATEBLUE("darkslateblue"), + + DARKSLATEGRAY("darkslategray"), + + DARKTURQUOISE("darkturquoise"), + + DARKVIOLET("darkviolet"), + + DEEPPINK("deeppink"), + + DEEPSKYBLUE("deepskyblue"), + + DIMGRAY("dimgray"), + + DODGERBLUE("dodgerblue"), + + FIREBRICK("firebrick"), + + FLORALWHITE("floralwhite"), + + FORESTGREEN("forestgreen"), + + FUCHSIA("fuchsia"), + + GAINSBORO("gainsboro"), + + GHOSTWHITE("ghostwhite"), + + GOLD("gold"), + + GOLDENROD("goldenrod"), + + GRAY("gray"), + + GREEN("green"), + + GREENYELLOW("greenyellow"), + + HONEYDEW("honeydew"), + + HOTPINK("hotpink"), + + INDIANRED("indianred"), + + INDIGO("indigo"), + + IVORY("ivory"), + + KHAKI("khaki"), + + LAVENDER("lavender"), + + LAVENDERBLUSH("lavenderblush"), + + LAWNGREEN("lawngreen"), + + LEMONCHIFFON("lemonchiffon"), + + LIGHTBLUE("lightblue"), + + LIGHTCORAL("lightcoral"), + + LIGHTCYAN("lightcyan"), + + LIGHTGOLDENRODYE("lightgoldenrodye"), + + LIGHTGREEN("lightgreen"), + + LIGHTGREY("lightgrey"), + + LIGHTPINK("lightpink"), + + LIGHTSALMON("lightsalmon"), + + LIGHTSEAGREEN("lightseagreen"), + + LIGHTSKYBLUE("lightskyblue"), + + LIGHTSLATEGRAY("lightslategray"), + + LIGHTSTEELBLUE("lightsteelblue"), + + LIGHTYELLOW("lightyellow"), + + LIME("lime"), + + LIMEGREEN("limegreen"), + + LINEN("linen"), + + MAGENTA("magenta"), + + MAROON("maroon"), + + MEDIUMAQUAMARINE("mediumaquamarine"), + + MEDIUMBLUE("mediumblue"), + + MEDIUMORCHID("mediumorchid"), + + MEDIUMPURPLE("mediumpurple"), + + MEDIUMSEAGREEN("mediumseagreen"), + + MEDIUMSLATEBLUE("mediumslateblue"), + + MEDIUMSPRINGGREE("mediumspringgree"), + + MEDIUMTURQUOISE("mediumturquoise"), + + MEDIUMVIOLETRED("mediumvioletred"), + + MIDNIGHTBLUE("midnightblue"), + + MINTCREAM("mintcream"), + + MISTYROSE("mistyrose"), + + MOCCASIN("moccasin"), + + NAVAJOWHITE("navajowhite"), + + NAVY("navy"), + + NAVYBLUE("navyblue"), + + OLDLACE("oldlace"), + + OLIVE("olive"), + + OLIVEDRAB("olivedrab"), + + ORANGE("orange"), + + ORANGERED("orangered"), + + ORCHID("orchid"), + + PALEGOLDENROD("palegoldenrod"), + + PALEGREEN("palegreen"), + + PALETURQUOISE("paleturquoise"), + + PALEVIOLETRED("palevioletred"), + + PAPAYAWHIP("papayawhip"), + + PEACHPUFF("peachpuff"), + + PERU("peru"), + + PINK("pink"), + + PLUM("plum"), + + POWDERBLUE("powderblue"), + + PURPLE("purple"), + + RED("red"), + + ROSYBROWN("rosybrown"), + + ROYALBLUE("royalblue"), + + SADDLEBROWN("saddlebrown"), + + SALMON("salmon"), + + SANDYBROWN("sandybrown"), + + SEAGREEN("seagreen"), + + SEASHELL("seashell"), + + SIENNA("sienna"), + + SILVER("silver"), + + SKYBLUE("skyblue"), + + SLATEBLUE("slateblue"), + + SLATEGRAY("slategray"), + + SNOW("snow"), + + SPRINGGREEN("springgreen"), + + STEELBLUE("steelblue"), + + TAN("tan"), + + TEAL("teal"), + + THISTLE("thistle"), + + TOMATO("tomato"), + + TURQUOISE("turquoise"), + + VIOLET("violet"), + + WHEAT("wheat"), + + WHITE("white"), + + WHITESMOKE("whitesmoke"), + + YELLOW("yellow"), + + YELLOWGREEN("yellowgreen"); + + private final String color; + + private GnuplotColor(final String color) { + this.color = color; + } + + public String getColor() { + return color; + } +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java new file mode 100644 index 0000000..d689b83 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -0,0 +1,43 @@ +package org.lucares.recommind.logs; + +import java.util.Collection; + +public class GnuplotFileGenerator { + + public String generate(final GnuplotSettings settings, final Collection dataSeries) { + + final StringBuilder result = new StringBuilder(); + + appendfln(result, "set terminal %s size %d,%d", settings.getTerminal(), settings.getWidth(), + settings.getHeight()); + + appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator()); + appendfln(result, "set timefmt '%s'", settings.getTimefmt()); + + appendfln(result, "set xdata time"); + appendfln(result, "set format x \"%s\"", settings.getFormatX()); + appendfln(result, "set xlabel \"%s\"", settings.getXlabel()); + appendfln(result, "set xtics rotate by %d", settings.getRotateXAxisLabel()); + + appendfln(result, "set ylabel \"%s\"", settings.getYlabel()); + + appendfln(result, "set output \"%s\"", settings.getOutput().getAbsolutePath()); + appendf(result, "plot "); + + for (final DataSeries dataSerie : dataSeries) { + appendfln(result, "'%s' using 1:2 title '%s' with points, \\", dataSerie.getDataFile(), + dataSerie.getTitle()); + } + + return result.toString(); + } + + private void appendfln(final StringBuilder builder, final String format, final Object... args) { + builder.append(String.format(format + "\n", args)); + } + + private void appendf(final StringBuilder builder, final String format, final Object... args) { + builder.append(String.format(format, args)); + } + +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java new file mode 100644 index 0000000..8645d88 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java @@ -0,0 +1,115 @@ +package org.lucares.recommind.logs; + +import java.io.File; + +public class GnuplotSettings { + private String terminal = "png"; + private int height = 1200; + private int width = 1600; + private String timefmt = "%Y-%m-%dT%H:%M:%S"; + + // set format x "%m-%d\n%H:%M" + private String formatX = "%Y-%m-%d %H:%M:%S"; + + // set datafile separator "," + private String datafileSeparator = ","; + + // set xlabel "Time" + private String xlabel = "Time"; + + // set ylabel "Traffic" + private String ylabel = "Duration in ms"; + + // set output "datausage.png" + private File output = new File("/tmp/out.png"); + + // set xtics rotate by 80 + private int rotateXAxisLabel = -80; + + public GnuplotSettings(final File output) { + this.output = output; + } + + public int getRotateXAxisLabel() { + return rotateXAxisLabel; + } + + public void setRotateXAxisLabel(final int rotateXAxisLabel) { + this.rotateXAxisLabel = rotateXAxisLabel; + } + + public String getTerminal() { + return terminal; + } + + public void setTerminal(final String terminal) { + this.terminal = terminal; + } + + public int getHeight() { + return height; + } + + public void setHeight(final int height) { + this.height = height; + } + + public int getWidth() { + return width; + } + + public void setWidth(final int width) { + this.width = width; + } + + public String getTimefmt() { + return timefmt; + } + + public void setTimefmt(final String timefmt) { + this.timefmt = timefmt; + } + + public String getFormatX() { + return formatX; + } + + public void setFormatX(final String formatX) { + this.formatX = formatX; + } + + public String getDatafileSeparator() { + return datafileSeparator; + } + + public void setDatafileSeparator(final String datafileSeparator) { + this.datafileSeparator = datafileSeparator; + } + + public String getXlabel() { + return xlabel; + } + + public void setXlabel(final String xlabel) { + this.xlabel = xlabel; + } + + public String getYlabel() { + return ylabel; + } + + public void setYlabel(final String ylabel) { + this.ylabel = ylabel; + } + + public File getOutput() { + return output; + } + + public void setOutput(final File output) { + this.output = output; + } + + // plot 'sample.txt' using 1:2 title 'Bytes' with linespoints 2 + +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java new file mode 100644 index 0000000..f8d3958 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -0,0 +1,73 @@ +package org.lucares.recommind.logs; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.stream.Stream; + +import org.lucares.performance.db.Entry; +import org.lucares.performance.db.FileUtils; +import org.lucares.performance.db.PerformanceDb; +import org.lucares.tanga.svak.StreamSvWriter; +import org.lucares.tanga.svak.StreamSvWriterSettings; +import org.lucares.tanga.svak.StreamSvWriterSettings.Escaping; +import org.lucares.tanga.svak.StreamSvWriterSettings.FieldQuoting; + +public class Plotter { + public static void main(final String[] args) throws Exception { + final Path dataDirectory = Paths.get(args[0]); + final Path outputDirectory = Paths.get(args[1]); + final String query = args[2]; + final Path tmpBaseDir = Paths.get(args[0], "tmp"); + Files.createDirectories(tmpBaseDir); + Files.createDirectories(outputDirectory); + final Path tmpDirectory = Files.createTempDirectory(tmpBaseDir, "gnuplot"); + try { + + final Collection dataSeries = new ArrayList<>(); + + try (PerformanceDb db = new PerformanceDb(dataDirectory)) { + final Stream entries = db.get(query); + + final File dataFile = File.createTempFile("data", ".dat", tmpDirectory.toFile()); + final DataSeries dataSerie = new DataSeries(dataFile, query); + toCsv(entries, dataFile); + + dataSeries.add(dataSerie); + } + + final File outputFile = File.createTempFile("out", ".png", outputDirectory.toFile()); + final Gnuplot gnuplot = new Gnuplot(tmpDirectory); + final GnuplotSettings settings = new GnuplotSettings(outputFile); + gnuplot.plot(settings, dataSeries); + } finally { + FileUtils.delete(tmpDirectory); + } + } + + private static void toCsv(final Stream entries, final File dataFile) throws IOException { + + final StreamSvWriterSettings svSettings = new StreamSvWriterSettings(",", "'"); + svSettings.setFieldQuoting(FieldQuoting.IF_NEEDED); + svSettings.setEscaping(Escaping.NONE); + try (FileOutputStream output = new FileOutputStream(dataFile); + StreamSvWriter writer = new StreamSvWriter(output, svSettings)) { + + final Iterator it = entries.iterator(); + while (it.hasNext()) { + final Entry entry = it.next(); + + final String value = String.valueOf(entry.getValue()); + final String date = entry.getDate().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); + writer.writeLine(date, value); + } + } + } +}