diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java index 1f12a1f..505ef37 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java @@ -260,10 +260,6 @@ public class DataStore { final Doc doc = docIdToDoc.get(docId); - if (!doc.getTags().getValue("pod").equals("vadtrans01")){ - System.out.println(); - } - result.add(doc); } } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java index 21ec47b..e8c6ce0 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java @@ -36,6 +36,8 @@ public class PlotSettings { private boolean keyOutside; + private PlotType plotType; + public String getQuery() { return query; } @@ -178,4 +180,12 @@ public class PlotSettings { public boolean isKeyOutside() { return keyOutside; } + + public void setPlotType(PlotType plotType) { + this.plotType = plotType; + } + + public PlotType getPlotType() { + return plotType; + } } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotType.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotType.java new file mode 100644 index 0000000..baaa9eb --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotType.java @@ -0,0 +1,5 @@ +package org.lucares.pdb.plot.api; + +public enum PlotType { + SCATTER, PERCENTILES +} diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java index c868071..323473c 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; import org.lucares.pdb.plot.api.AggregatedData; +import org.lucares.pdb.plot.api.Limit; public class DataSeries { public static final Comparator BY_NUMBER_OF_VALUES = ( @@ -66,5 +67,38 @@ public class DataSeries { return result; } + static Comparator getDataSeriesComparator(final Limit limitBy) { + + switch (limitBy) { + case MOST_VALUES: + return DataSeries.BY_NUMBER_OF_VALUES.reversed(); + case FEWEST_VALUES: + return DataSeries.BY_NUMBER_OF_VALUES; + case MAX_VALUE: + return DataSeries.BY_MAX_VALUE.reversed(); + case MIN_VALUE: + return DataSeries.BY_MAX_VALUE; + case NO_LIMIT: + return DataSeries.BY_NUMBER_OF_VALUES; + } + throw new IllegalStateException("unhandled enum: "+ limitBy); + } + + static void sortAndLimit(final List dataSeries, final Limit limitBy, final int limit) { + + dataSeries.sort(DataSeries.getDataSeriesComparator(limitBy)); + + switch (limitBy) { + case FEWEST_VALUES: + case MOST_VALUES: + case MAX_VALUE: + case MIN_VALUE: + while (dataSeries.size() > limit) { + dataSeries.remove(limit); + } + break; + case NO_LIMIT: + } + } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java index b89fc7d..36b01d8 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -21,11 +21,14 @@ public class GnuplotFileGenerator { appendfln(result, "set timefmt '%s'", settings.getTimefmt()); - appendfln(result, "set xdata time"); - appendfln(result, "set format x \"%s\"", settings.getFormatX()); - appendfln(result, "set xlabel \"%s\"", settings.getXlabel()); - appendfln(result, "set xtics rotate by %d", settings.getRotateXAxisLabel()); - appendfln(result, "set xrange [\"%s\":\"%s\"]", settings.getDateFrom(), settings.getDateTo()); + final XAxisSettings xAxis = settings.getxAxisSettings(); + if (xAxis.isxDataTime()){ + appendfln(result, "set xdata time"); + } + appendfln(result, "set format x \"%s\"", xAxis.getFormatX()); + appendfln(result, "set xlabel \"%s\"", xAxis.getXlabel()); + appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel()); + appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo()); final long graphOffset = settings.getYAxisScale() == AxisScale.LINEAR ? 0 : 1; appendfln(result, "set yrange [\""+graphOffset+"\":]"); @@ -46,8 +49,8 @@ public class GnuplotFileGenerator { appendfln(result, "set output \"%s\"", settings.getOutput().toAbsolutePath().toString().replace("\\", "/")); // marker lines that show which area will be zoomed - final long minDate = Long.parseLong(settings.getDateFrom()); - final long maxDate = Long.parseLong(settings.getDateTo()); + final long minDate = Long.parseLong(settings.getxAxisSettings().getFrom()); + final long maxDate = Long.parseLong(settings.getxAxisSettings().getTo()); appendfln(result, "set arrow from "+(minDate + (maxDate-minDate)*0.25)+","+graphOffset+" rto graph 0,1 lt 3 lc rgb \"#EEEEEE\" nohead"); appendfln(result, "set arrow from "+(minDate + (maxDate-minDate)*0.75)+","+graphOffset+" rto graph 0,1 lc rgb \"#EEEEEE\" nohead"); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java index 551a10d..cc4e7bd 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java @@ -9,16 +9,12 @@ public class GnuplotSettings { private String terminal = "png"; private int height = 1200; private int width = 1600; - private String timefmt = "%s"; //"%Y-%m-%dT%H:%M:%S"; // TODO @ahr timefmt + private String timefmt = "%s"; // time as unix epoch, but as double - // set format for x-axis - private String formatX = "%Y-%m-%d %H:%M:%S"; // set datafile separator private String datafileSeparator = ","; - // set xlabel - private String xlabel = "Time"; // set ylabel private String ylabel = "Duration in ms"; @@ -26,26 +22,30 @@ public class GnuplotSettings { // set output "datausage.png" private final Path output; - // set xtics rotate by 10 degree - private int rotateXAxisLabel = -10; private AxisScale yAxisScale; - private String dateFrom; - private String dateTo; private AggregateHandler aggregate; private boolean keyOutside = false; + + private XAxisSettings xAxisSettings = new XAxisSettings(); public GnuplotSettings(final Path output) { this.output = output; } - public int getRotateXAxisLabel() { - return rotateXAxisLabel; + + + public XAxisSettings getxAxisSettings() { + return xAxisSettings; } - public void setRotateXAxisLabel(final int rotateXAxisLabel) { - this.rotateXAxisLabel = rotateXAxisLabel; + + + public void setxAxisSettings(XAxisSettings xAxisSettings) { + this.xAxisSettings = xAxisSettings; } + + public String getTerminal() { return terminal; } @@ -78,13 +78,6 @@ public class GnuplotSettings { this.timefmt = timefmt; } - public String getFormatX() { - return formatX; - } - - public void setFormatX(final String formatX) { - this.formatX = formatX; - } public String getDatafileSeparator() { return datafileSeparator; @@ -94,14 +87,6 @@ public class GnuplotSettings { this.datafileSeparator = datafileSeparator; } - public String getXlabel() { - return xlabel; - } - - public void setXlabel(final String xlabel) { - this.xlabel = xlabel; - } - public String getYlabel() { return ylabel; } @@ -122,22 +107,6 @@ public class GnuplotSettings { return yAxisScale; } - public void setDateFrom(final String dateFrom) { - this.dateFrom = dateFrom; - } - - public String getDateFrom() { - return dateFrom; - } - - public void setDateTo(final String dateTo) { - this.dateTo = dateTo; - } - - public String getDateTo() { - return dateTo; - } - public void setAggregate(AggregateHandler aggregate) { this.aggregate = aggregate; } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java index d362f5d..ea21780 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -1,57 +1,14 @@ package org.lucares.recommind.logs; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Formatter; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import org.lucares.pdb.api.Entry; -import org.lucares.pdb.api.GroupResult; -import org.lucares.pdb.api.Result; -import org.lucares.pdb.api.Tags; -import org.lucares.pdb.plot.api.CustomAggregator; -import org.lucares.pdb.plot.api.Limit; import org.lucares.pdb.plot.api.PlotSettings; +import org.lucares.pdb.plot.api.PlotType; import org.lucares.performance.db.PerformanceDb; -import org.lucares.utils.file.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class Plotter { - private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class); - private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter"); - - private static final String DEFAULT_GROUP = ""; - - private static final int INT_TO_STRING_CACHE_SIZE= 1000; - private static final String[] INT_TO_STRING; - static { - - INT_TO_STRING = new String[INT_TO_STRING_CACHE_SIZE]; - - for (int i = 0; i < INT_TO_STRING_CACHE_SIZE; i++){ - INT_TO_STRING[i] = String.valueOf(i); - } - } private final PerformanceDb db; private final Path tmpBaseDir; @@ -75,227 +32,23 @@ public class Plotter { } public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { - - LOGGER.trace("start plot: {}", plotSettings); + PlotType plotType = plotSettings.getPlotType(); + + switch (plotType) { + case SCATTER: + final ScatterPlot scatterPlot = new ScatterPlot(db, tmpBaseDir, outputDir); + return scatterPlot.plot(plotSettings); - final String tmpSubDir = uniqueDirectoryName(); - final Path tmpDir = tmpBaseDir.resolve(tmpSubDir); - try { - Files.createDirectories(tmpDir); - final List dataSeries = Collections.synchronizedList(new ArrayList<>()); + default: + throw new UnsupportedOperationException("plot of type " + plotType + " not supported."); + } + + } + + - final String query = plotSettings.getQuery(); - final List groupBy = plotSettings.getGroupBy(); - final int height = plotSettings.getHeight(); - final int width = plotSettings.getWidth(); - final OffsetDateTime dateFrom = plotSettings.dateFrom(); - final OffsetDateTime dateTo = plotSettings.dateTo(); - - final Result result = db.get(query, groupBy); - - final long start = System.nanoTime(); - final AtomicInteger idCounter = new AtomicInteger(0); - result.getGroups().stream().parallel().forEach(groupResult -> { - try{ - final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings); - final int id = idCounter.getAndIncrement(); - final String title = title(groupResult.getGroupedBy(), csvSummary.getValues()); - final DataSeries dataSerie = new DataSeries("id"+id, title, csvSummary); - if (dataSerie.getValues() > 0) { - dataSeries.add(dataSerie); - } - }catch (Exception e){ - throw new IllegalStateException( e); // TODO handle - } - }); - METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 - + "ms"); - if (dataSeries.isEmpty()) { - throw new NoDataPointsException(); - } - - sortAndLimit(dataSeries, plotSettings); - - final Path outputFile = Files.createTempFile(outputDir, "out", ".png"); - final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); - final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile); - gnuplotSettings.setHeight(height); - gnuplotSettings.setWidth(width); - defineXAxis(gnuplotSettings, plotSettings.dateFrom(), plotSettings.dateTo()); - - gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); - gnuplotSettings.setAggregate(plotSettings.getAggregate()); - gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); - gnuplot.plot(gnuplotSettings, dataSeries); - - return new PlotResult(outputFile.getFileName(), dataSeries); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Plotting was interrupted."); - } catch (final IOException e) { - throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); - } finally { - FileUtils.delete(tmpDir); - LOGGER.trace("done plot"); - } - } - - private void defineXAxis(final GnuplotSettings gnuplotSettings, final OffsetDateTime minDate, - final OffsetDateTime maxDate) { - - String formatX; - int rotateX; - String formattedMinDate; - String formattedMaxDate; - if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) { - formatX = "%Y-%m-%d"; - rotateX = 0; - } else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) { - formatX = "%Y-%m-%d %H:%M:%S"; - rotateX = gnuplotSettings.getRotateXAxisLabel(); - } else { - formatX = "%Y-%m-%d %H:%M:%.3S"; - rotateX = gnuplotSettings.getRotateXAxisLabel(); - } - formattedMinDate = String.valueOf(minDate.toEpochSecond()); - formattedMaxDate = String.valueOf(maxDate.toEpochSecond()); - - gnuplotSettings.setFormatX(formatX); - gnuplotSettings.setRotateXAxisLabel(rotateX); - gnuplotSettings.setDateFrom(formattedMinDate); - gnuplotSettings.setDateTo(formattedMaxDate); - } - - private String uniqueDirectoryName() { - return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_" - + UUID.randomUUID().toString(); - } - - private void sortAndLimit(final List dataSeries, final PlotSettings plotSettings) { - - final Limit limitBy = plotSettings.getLimitBy(); - dataSeries.sort(getDataSeriesComparator(limitBy)); - - switch (limitBy) { - case FEWEST_VALUES: - case MOST_VALUES: - case MAX_VALUE: - case MIN_VALUE: - while (dataSeries.size() > plotSettings.getLimit()) { - dataSeries.remove(plotSettings.getLimit()); - } - break; - case NO_LIMIT: - } - } - - private Comparator getDataSeriesComparator(final Limit limitBy) { - - switch (limitBy) { - case MOST_VALUES: - return DataSeries.BY_NUMBER_OF_VALUES.reversed(); - case FEWEST_VALUES: - return DataSeries.BY_NUMBER_OF_VALUES; - case MAX_VALUE: - return DataSeries.BY_MAX_VALUE.reversed(); - case MIN_VALUE: - return DataSeries.BY_MAX_VALUE; - case NO_LIMIT: - return DataSeries.BY_NUMBER_OF_VALUES; - } - throw new IllegalStateException("unhandled enum: "+ limitBy); - } - - private String title(final Tags tags, final int values) { - - final StringBuilder result = new StringBuilder(); - - if (tags.isEmpty()) { - result.append(DEFAULT_GROUP); - } else { - tags.forEach((k, v) -> { - if (result.length() > 0) { - result.append(" / "); - } - result.append(v); - }); - } - - result.append(" ("); - result.append(String.format("%,d", values)); - result.append(")"); - - return result.toString(); - - } - - private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom, - final OffsetDateTime dateTo, PlotSettings plotSettings) throws IOException { - - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - final long start = System.nanoTime(); - final Stream entries = groupResult.asStream(); - int count = 0; - final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); - final long toEpochMilli = dateTo.toInstant().toEpochMilli(); - final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); - final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli); - - long maxValue = 0; - long ignoredValues = 0; - final int separator = ','; - final int newline = '\n'; - final StringBuilder formattedDateBuilder = new StringBuilder(); - try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII)); - final Formatter formatter = new Formatter(formattedDateBuilder);) { - - final Iterator it = entries.iterator(); - while (it.hasNext()) { - final Entry entry = it.next(); - - long epochMilli = entry.getEpochMilli(); - if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) { - - long value = entry.getValue(); - final String stringValue = longToString(value); - final String formattedDate; - - if (useMillis){ - formattedDateBuilder.delete(0, formattedDateBuilder.length()); - formatter.format("%.3f", epochMilli / 1000.0); - formattedDate = formattedDateBuilder.toString(); - }else { - formattedDate = String.valueOf(epochMilli / 1000); - } - - output.write(formattedDate); - output.write(separator); - output.write(stringValue); - output.write(newline); - - aggregator.addValue(epochMilli, value); - - count++; - maxValue = Math.max(maxValue, value); - }else { - ignoredValues++; - } - } - } - - METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy(),dataFile); - return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData()); - } - private static String longToString(final long value){ - // using pre-generated strings reduces memory allocation by up to 25% - - if (value < INT_TO_STRING_CACHE_SIZE){ - return INT_TO_STRING[(int) value]; - } - return String.valueOf(value); - } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java new file mode 100644 index 0000000..00d7fb6 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java @@ -0,0 +1,270 @@ +package org.lucares.recommind.logs; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Formatter; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import org.lucares.pdb.api.Entry; +import org.lucares.pdb.api.GroupResult; +import org.lucares.pdb.api.Result; +import org.lucares.pdb.api.Tags; +import org.lucares.pdb.plot.api.CustomAggregator; +import org.lucares.pdb.plot.api.Limit; +import org.lucares.pdb.plot.api.PlotSettings; +import org.lucares.performance.db.PerformanceDb; +import org.lucares.utils.file.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScatterPlot { + + private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class); + private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter"); + + private static final String DEFAULT_GROUP = ""; + + private static final int INT_TO_STRING_CACHE_SIZE= 1000; + private static final String[] INT_TO_STRING; + static { + + INT_TO_STRING = new String[INT_TO_STRING_CACHE_SIZE]; + + for (int i = 0; i < INT_TO_STRING_CACHE_SIZE; i++){ + INT_TO_STRING[i] = String.valueOf(i); + } + } + + private final PerformanceDb db; + private final Path tmpBaseDir; + private final Path outputDir; + + public ScatterPlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) { + this.db = db; + this.tmpBaseDir = tmpBaseDir; + this.outputDir = outputDir; + + if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) { + throw new IllegalArgumentException(tmpBaseDir + " is not a directory"); + } + if (!Files.isDirectory(outputDir)) { + throw new IllegalArgumentException(outputDir + " is not a directory"); + } + } + + public Path getOutputDir() { + return outputDir; + } + + public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { + + LOGGER.trace("start plot: {}", plotSettings); + + + final String tmpSubDir = uniqueDirectoryName(); + final Path tmpDir = tmpBaseDir.resolve(tmpSubDir); + try { + Files.createDirectories(tmpDir); + final List dataSeries = Collections.synchronizedList(new ArrayList<>()); + + final String query = plotSettings.getQuery(); + final List groupBy = plotSettings.getGroupBy(); + final int height = plotSettings.getHeight(); + final int width = plotSettings.getWidth(); + final OffsetDateTime dateFrom = plotSettings.dateFrom(); + final OffsetDateTime dateTo = plotSettings.dateTo(); + + final Result result = db.get(query, groupBy); + + final long start = System.nanoTime(); + final AtomicInteger idCounter = new AtomicInteger(0); + result.getGroups().stream().parallel().forEach(groupResult -> { + try{ + final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings); + + final int id = idCounter.getAndIncrement(); + final String title = title(groupResult.getGroupedBy(), csvSummary.getValues()); + final DataSeries dataSerie = new DataSeries("id"+id, title, csvSummary); + if (dataSerie.getValues() > 0) { + dataSeries.add(dataSerie); + } + }catch (Exception e){ + throw new IllegalStateException( e); // TODO handle + } + }); + METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + + "ms"); + + if (dataSeries.isEmpty()) { + throw new NoDataPointsException(); + } + + final Limit limitBy = plotSettings.getLimitBy(); + int limit = plotSettings.getLimit(); + DataSeries.sortAndLimit(dataSeries, limitBy, limit); + + final Path outputFile = Files.createTempFile(outputDir, "out", ".png"); + final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); + final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile); + gnuplotSettings.setHeight(height); + gnuplotSettings.setWidth(width); + defineXAxis(gnuplotSettings, plotSettings.dateFrom(), plotSettings.dateTo()); + + gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); + gnuplotSettings.setAggregate(plotSettings.getAggregate()); + gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); + gnuplot.plot(gnuplotSettings, dataSeries); + + return new PlotResult(outputFile.getFileName(), dataSeries); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Plotting was interrupted."); + } catch (final IOException e) { + throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); + } finally { + FileUtils.delete(tmpDir); + LOGGER.trace("done plot"); + } + } + + private void defineXAxis(final GnuplotSettings gnuplotSettings, final OffsetDateTime minDate, + final OffsetDateTime maxDate) { + + String formatX; + int rotateX; + String formattedMinDate; + String formattedMaxDate; + if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) { + formatX = "%Y-%m-%d"; + rotateX = 0; + } else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) { + formatX = "%Y-%m-%d %H:%M:%S"; + rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); + } else { + formatX = "%Y-%m-%d %H:%M:%.3S"; + rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); + } + formattedMinDate = String.valueOf(minDate.toEpochSecond()); + formattedMaxDate = String.valueOf(maxDate.toEpochSecond()); + + gnuplotSettings.getxAxisSettings().setFormatX(formatX); + gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX); + gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate); + gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate); + } + + private String uniqueDirectoryName() { + return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_" + + UUID.randomUUID().toString(); + } + + + + private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom, + final OffsetDateTime dateTo, PlotSettings plotSettings) throws IOException { + + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + final long start = System.nanoTime(); + final Stream entries = groupResult.asStream(); + int count = 0; + final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); + final long toEpochMilli = dateTo.toInstant().toEpochMilli(); + final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); + final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli); + + long maxValue = 0; + long ignoredValues = 0; + final int separator = ','; + final int newline = '\n'; + final StringBuilder formattedDateBuilder = new StringBuilder(); + try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII)); + final Formatter formatter = new Formatter(formattedDateBuilder);) { + + final Iterator it = entries.iterator(); + while (it.hasNext()) { + final Entry entry = it.next(); + + long epochMilli = entry.getEpochMilli(); + if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) { + + long value = entry.getValue(); + final String stringValue = longToString(value); + final String formattedDate; + + if (useMillis){ + formattedDateBuilder.delete(0, formattedDateBuilder.length()); + formatter.format("%.3f", epochMilli / 1000.0); + formattedDate = formattedDateBuilder.toString(); + }else { + formattedDate = String.valueOf(epochMilli / 1000); + } + + output.write(formattedDate); + output.write(separator); + output.write(stringValue); + output.write(newline); + + aggregator.addValue(epochMilli, value); + + count++; + maxValue = Math.max(maxValue, value); + }else { + ignoredValues++; + } + } + } + + METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy(),dataFile); + return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData()); + } + + private static String longToString(final long value){ + // using pre-generated strings reduces memory allocation by up to 25% + + if (value < INT_TO_STRING_CACHE_SIZE){ + return INT_TO_STRING[(int) value]; + } + return String.valueOf(value); + } + + private String title(final Tags tags, final int values) { + + final StringBuilder result = new StringBuilder(); + + if (tags.isEmpty()) { + result.append(DEFAULT_GROUP); + } else { + tags.forEach((k, v) -> { + if (result.length() > 0) { + result.append(" / "); + } + result.append(v); + }); + } + + result.append(" ("); + result.append(String.format("%,d", values)); + result.append(")"); + + return result.toString(); + + } +} diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/XAxisSettings.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/XAxisSettings.java new file mode 100644 index 0000000..65e9f5c --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/XAxisSettings.java @@ -0,0 +1,76 @@ +package org.lucares.recommind.logs; + +public class XAxisSettings { + + // set xdata time + private boolean xDataTime = true; + + // set format for x-axis + private String formatX = "%Y-%m-%d %H:%M:%S"; + + // set xlabel + private String xlabel = "Time"; + + // set xtics rotate by 10 degree + private int rotateXAxisLabel = -10; + + private String from; + private String to; + + public boolean isxDataTime() { + return xDataTime; + } + + public void setxDataTime(boolean xDataTime) { + this.xDataTime = xDataTime; + } + + public String getFormatX() { + return formatX; + } + + public void setFormatX(String formatX) { + this.formatX = formatX; + } + + public String getXlabel() { + return xlabel; + } + + public void setXlabel(String xlabel) { + this.xlabel = xlabel; + } + + public int getRotateXAxisLabel() { + return rotateXAxisLabel; + } + + public void setRotateXAxisLabel(int rotateXAxisLabel) { + this.rotateXAxisLabel = rotateXAxisLabel; + } + + public String getFrom() { + return from; + } + + public void setFrom(String from) { + this.from = from; + } + + public String getTo() { + return to; + } + + public void setTo(String to) { + this.to = to; + } + + @Override + public String toString() { + return "XAxisSettings [xDataTime=" + xDataTime + ", formatX=" + formatX + + ", xlabel=" + xlabel + ", rotateXAxisLabel=" + + rotateXAxisLabel + ", from=" + from + ", to=" + to + "]"; + } + + +} diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 99f74dc..9aa51d8 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -22,6 +22,7 @@ class PlotSettingsTransformer { result.setDateFrom(request.getDateFrom()); result.setDateRange(request.getDateRange()); result.setYAxisScale(request.getAxisScale()); + result.setPlotType(request.getPlotType()); result.setAggregate(toAggregateInternal(request.getAggregate())); result.setKeyOutside(request.isKeyOutside()); diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java b/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java index 96cff21..e0b3ce1 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java @@ -4,6 +4,7 @@ import java.util.List; import org.lucares.pdb.plot.api.AxisScale; import org.lucares.pdb.plot.api.Limit; +import org.lucares.pdb.plot.api.PlotType; public class PlotRequest { private String query; @@ -24,6 +25,8 @@ public class PlotRequest { private String dateRange; + private PlotType plotType = PlotType.SCATTER; + private Aggregate aggregate = Aggregate.NONE; private boolean keyOutside; @@ -108,6 +111,14 @@ public class PlotRequest { this.yAxis = yAxis; } + public PlotType getPlotType() { + return plotType; + } + + public void setPlotType(PlotType plotType) { + this.plotType = plotType; + } + public void setAggregate(Aggregate aggregate) { this.aggregate = aggregate; } diff --git a/pdb-ui/src/main/resources/resources/js/search.js b/pdb-ui/src/main/resources/resources/js/search.js index 36a24db..b7f1e8a 100644 --- a/pdb-ui/src/main/resources/resources/js/search.js +++ b/pdb-ui/src/main/resources/resources/js/search.js @@ -16,6 +16,9 @@ $(document).ready(function(){ $('#search-limit-by').change(updateSearchLimitValue); disableSplitBy(); + $('#plot-type').change(updatePlotType); + updatePlotType(); + $('#nav_left').click(dateLeftShift); $('#nav_left_half').click(dateHalfLeftShift); $('#nav_right_half').click(dateHalfRightShift); @@ -252,6 +255,16 @@ function updateSearchLimitValue () { } } +function updatePlotType(){ + var optionSelected = $('#plot-type').find("option:selected"); + var valueSelected = optionSelected.val(); + if (valueSelected == "PERCENTILES"){ + $('#group-show-aggregate').hide(); + }else{ + $('#group-show-aggregate').show(); + } +} + function enableSplitBy(fieldValues) { splitBy['field'] = $('#split-by').val(); splitBy['values'] = fieldValues; @@ -399,6 +412,7 @@ function sendPlotRequest(query){ request['dateFrom'] = $('#search-date-from').val(); request['dateRange'] = $('#search-date-range').val(); request['axisScale'] = $('#search-y-axis-scale').val(); + request['plotType'] = $('#plot-type').val(); request['aggregate'] = $('#show-aggregate').val(); request['keyOutside'] = $('#key-outside').is(":checked"); diff --git a/pdb-ui/src/main/resources/templates/main.html b/pdb-ui/src/main/resources/templates/main.html index 473a0b8..7ec522f 100644 --- a/pdb-ui/src/main/resources/templates/main.html +++ b/pdb-ui/src/main/resources/templates/main.html @@ -15,8 +15,6 @@ - -
@@ -70,6 +68,13 @@
+ + +
+