From 7c61686808ac0abe9e99ed943a4a1e9be382404e Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 19 Oct 2019 19:01:56 +0200 Subject: [PATCH] merge ScatterPlot into Plotter --- .../org/lucares/recommind/logs/Plotter.java | 347 +++++++++++++- .../lucares/recommind/logs/ScatterPlot.java | 435 ------------------ 2 files changed, 326 insertions(+), 456 deletions(-) delete mode 100644 pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java index d7f00f5..930388b 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -1,40 +1,345 @@ package org.lucares.recommind.logs; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Formatter; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.lucares.collections.LongList; +import org.lucares.collections.Sparse2DLongArray; +import org.lucares.pdb.api.DateTimeRange; +import org.lucares.pdb.api.GroupResult; +import org.lucares.pdb.api.Query; +import org.lucares.pdb.api.Result; +import org.lucares.pdb.api.Tags; +import org.lucares.pdb.plot.api.AxisScale; +import org.lucares.pdb.plot.api.CustomAggregator; +import org.lucares.pdb.plot.api.Limit; import org.lucares.pdb.plot.api.PlotSettings; +import org.lucares.pdb.plot.api.TimeRangeUnitInternal; import org.lucares.performance.db.PerformanceDb; +import org.lucares.utils.file.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class Plotter { - private final PerformanceDb db; - private final Path tmpBaseDir; - private final Path outputDir; + private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class); + private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter"); - public Plotter(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) { - this.db = db; - this.tmpBaseDir = tmpBaseDir; - this.outputDir = outputDir; + static final String DEFAULT_GROUP = ""; - if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) { - throw new IllegalArgumentException(tmpBaseDir + " is not a directory"); - } - if (!Files.isDirectory(outputDir)) { - throw new IllegalArgumentException(outputDir + " is not a directory"); - } - } + private final PerformanceDb db; + private final Path tmpBaseDir; + private final Path outputDir; - public Path getOutputDir() { - return outputDir; - } + public Plotter(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) { + this.db = db; + this.tmpBaseDir = tmpBaseDir; + this.outputDir = outputDir; - public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { + if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) { + throw new IllegalArgumentException(tmpBaseDir + " is not a directory"); + } + if (!Files.isDirectory(outputDir)) { + throw new IllegalArgumentException(outputDir + " is not a directory"); + } + } - final ScatterPlot plotter = new ScatterPlot(db, tmpBaseDir, outputDir); + public Path getOutputDir() { + return outputDir; + } - return plotter.plot(plotSettings); - } + public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { + + LOGGER.trace("start plot: {}", plotSettings); + + final String tmpSubDir = uniqueDirectoryName(); + final Path tmpDir = tmpBaseDir.resolve(tmpSubDir); + try { + Files.createDirectories(tmpDir); + final List dataSeries = Collections.synchronizedList(new ArrayList<>()); + + final String query = plotSettings.getQuery(); + final List groupBy = plotSettings.getGroupBy(); + final int height = plotSettings.getHeight(); + final int width = plotSettings.getWidth(); + final DateTimeRange dateRange = plotSettings.dateRange(); + final OffsetDateTime dateFrom = dateRange.getStart(); + final OffsetDateTime dateTo = dateRange.getEnd(); + + final Result result = db.get(new Query(query, dateRange), groupBy); + + final long start = System.nanoTime(); + final AtomicInteger idCounter = new AtomicInteger(0); + result.getGroups().stream().parallel().forEach(groupResult -> { + try { + final CsvSummary csvSummary = toCsvDeduplicated(groupResult, tmpDir, dateFrom, dateTo, plotSettings); + + final int id = idCounter.incrementAndGet(); + final String title = title(groupResult.getGroupedBy(), csvSummary); + final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.Points); + if (dataSerie.getValues() > 0) { + dataSeries.add(dataSerie); + } + } catch (final Exception e) { + throw new IllegalStateException(e); + } + }); + METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + + if (dataSeries.isEmpty()) { + throw new NoDataPointsException(); + } + + final Limit limitBy = plotSettings.getLimitBy(); + final int limit = plotSettings.getLimit(); + DataSeries.sortAndLimit(dataSeries, limitBy, limit); + DataSeries.setColors(dataSeries); + + final Path outputFile = Files.createTempFile(outputDir, "", ".png"); + { + final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); + final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile); + gnuplotSettings.setHeight(height); + gnuplotSettings.setWidth(width); + defineXAxis(gnuplotSettings, plotSettings.dateRange()); + + gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); + gnuplotSettings.setAggregate(plotSettings.getAggregate()); + defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(), + plotSettings.getYRangeUnit()); + gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); + gnuplot.plot(gnuplotSettings, dataSeries); + } + + final Path thumbnail; + if (plotSettings.isGenerateThumbnail()) { + thumbnail = Files.createTempFile(outputDir, "", ".png"); + final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); + final GnuplotSettings gnuplotSettings = new GnuplotSettings(thumbnail); + gnuplotSettings.setHeight(plotSettings.getThumbnailMaxHeight()); + gnuplotSettings.setWidth(plotSettings.getThumbnailMaxWidth()); + defineXAxis(gnuplotSettings, plotSettings.dateRange()); + + gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); + gnuplotSettings.setAggregate(plotSettings.getAggregate()); + defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(), + plotSettings.getYRangeUnit()); + gnuplotSettings.setKeyOutside(false); + gnuplotSettings.renderLabels(false); + gnuplot.plot(gnuplotSettings, dataSeries); + } else { + thumbnail = null; + } + + return new PlotResult(outputFile, dataSeries, thumbnail); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Plotting was interrupted."); + } catch (final IOException e) { + throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); + } finally { + FileUtils.delete(tmpDir); + LOGGER.trace("done plot"); + } + } + + private void defineYRange(final GnuplotSettings gnuplotSettings, final int yRangeMin, final int yRangeMax, + final TimeRangeUnitInternal yRangeUnit) { + + if (yRangeUnit != TimeRangeUnitInternal.AUTOMATIC) { + final int min = yRangeUnit.toMilliSeconds(yRangeMin); + final int max = yRangeUnit.toMilliSeconds(yRangeMax); + gnuplotSettings.setYRange(min, max); + } + } + + private void defineXAxis(final GnuplotSettings gnuplotSettings, final DateTimeRange dateTimeRange) { + + final OffsetDateTime minDate = dateTimeRange.getStart(); + final OffsetDateTime maxDate = dateTimeRange.getEnd(); + String formatX; + int rotateX; + String formattedMinDate; + String formattedMaxDate; + if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) { + formatX = "%Y-%m-%d"; + rotateX = 0; + } else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) { + formatX = "%Y-%m-%d\\n%H:%M:%S"; + rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); + } else { + formatX = "%Y-%m-%d\\n%H:%M:%.3S"; + rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); + } + formattedMinDate = String.valueOf(minDate.toEpochSecond()); + formattedMaxDate = String.valueOf(maxDate.toEpochSecond()); + + gnuplotSettings.getxAxisSettings().setFormatX(formatX); + gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX); + gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate); + gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate); + } + + private static CsvSummary toCsvDeduplicated(final GroupResult groupResult, final Path tmpDir, + final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException { + + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + final long start = System.nanoTime(); + final Stream timeValueStream = groupResult.asStream(); + final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); + final long toEpochMilli = dateTo.toInstant().toEpochMilli(); + final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); + final long plotAreaWidthInPx = plotSettings.getWidth() - GnuplotSettings.GNUPLOT_LEFT_RIGHT_MARGIN; + final long plotAreaHeightInPx = plotSettings.getHeight() - GnuplotSettings.GNUPLOT_TOP_BOTTOM_MARGIN; + final long epochMillisPerPixel = Math.max(1, (toEpochMilli - fromEpochMilli) / plotAreaWidthInPx); + + final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0 + : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin()); + final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE + : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax()); + final long durationMillisPerPixel = plotSettings.getYAxisScale() == AxisScale.LINEAR + ? Math.max(1, (maxValue - minValue) / plotAreaHeightInPx) + : 1; + + final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli, + toEpochMilli); + + final Sparse2DLongArray matrix2d = new Sparse2DLongArray(); + int count = 0; // number of values in the x-axis range (used to compute stats) + int plottedValues = 0; + long statsMaxValue = 0; + double statsCurrentAverage = 0.0; + long ignoredValues = 0; + final int separator = ','; + final int newline = '\n'; + + final Iterator it = timeValueStream.iterator(); + while (it.hasNext()) { + final LongList entry = it.next(); + + for (int i = 0; i < entry.size(); i += 2) { + + final long epochMilli = entry.get(i); + if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) { + ignoredValues++; + continue; + } + + final long value = entry.get(i + 1); + + aggregator.addValue(epochMilli, value); + + // compute stats + count++; + statsMaxValue = Math.max(statsMaxValue, value); + + // compute average (important to do this after 'count' has been incremented) + statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count; + + // check if value is in the selected y-range + if (value < minValue || value > maxValue) { + ignoredValues++; + continue; + } + + final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel; + final long roundedValue = value - value % durationMillisPerPixel; + matrix2d.put(roundedEpochMilli, roundedValue, 1); + + plottedValues++; + } + } + + long[] actualValuesWritten = new long[1]; + final StringBuilder formattedDateBuilder = new StringBuilder(); + try ( + final LambdaFriendlyWriter output = new LambdaFriendlyWriter( + new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1))); + final Formatter formatter = new Formatter(formattedDateBuilder);) { + + matrix2d.forEach((epochMilli, value, __) -> { + + final String stringValue = LongUtils.longToString(value); + final String formattedDate; + + if (useMillis) { + formattedDateBuilder.delete(0, formattedDateBuilder.length()); + formatter.format("%.3f", epochMilli / 1000.0); + formattedDate = formattedDateBuilder.toString(); + } else { + formattedDate = String.valueOf(epochMilli / 1000); + } + + output.write(formattedDate); + output.write(separator); + output.write(stringValue); + output.write(newline); + actualValuesWritten[0]++; + }); + } + + METRICS_LOGGER.debug( + "wrote {} (actual: {} factor: {}%) values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", + actualValuesWritten[0], count, (double) count / (actualValuesWritten[0]), + (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), + groupResult.getGroupedBy().asString(), dataFile); + return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage, + aggregator.getAggregatedData()); + + } + + static String uniqueDirectoryName() { + return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_" + + UUID.randomUUID().toString(); + } + + static String title(final Tags tags, final CsvSummary csvSummary) { + + final StringBuilder result = new StringBuilder(); + + final int values = csvSummary.getValues(); + final int plottedValues = csvSummary.getPlottedValues(); + + if (tags.isEmpty()) { + result.append(DEFAULT_GROUP); + } else { + tags.forEach((k, v) -> { + if (result.length() > 0) { + result.append(" / "); + } + result.append(v); + }); + } + + result.append(" ("); + if (plottedValues != values) { + result.append(String.format("%,d / %,d", plottedValues, values)); + } else { + result.append(String.format("%,d", values)); + } + result.append(")"); + + return result.toString(); + + } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java deleted file mode 100644 index a59b371..0000000 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java +++ /dev/null @@ -1,435 +0,0 @@ -package org.lucares.recommind.logs; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Formatter; -import java.util.Iterator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; - -import org.apache.commons.lang3.math.NumberUtils; -import org.lucares.collections.LongList; -import org.lucares.collections.Sparse2DLongArray; -import org.lucares.pdb.api.DateTimeRange; -import org.lucares.pdb.api.GroupResult; -import org.lucares.pdb.api.Query; -import org.lucares.pdb.api.Result; -import org.lucares.pdb.api.Tags; -import org.lucares.pdb.plot.api.AxisScale; -import org.lucares.pdb.plot.api.CustomAggregator; -import org.lucares.pdb.plot.api.Limit; -import org.lucares.pdb.plot.api.PlotSettings; -import org.lucares.pdb.plot.api.TimeRangeUnitInternal; -import org.lucares.performance.db.PerformanceDb; -import org.lucares.utils.file.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ScatterPlot { - - private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class); - private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter"); - - static final String DEFAULT_GROUP = ""; - - private final PerformanceDb db; - private final Path tmpBaseDir; - private final Path outputDir; - - public ScatterPlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) { - this.db = db; - this.tmpBaseDir = tmpBaseDir; - this.outputDir = outputDir; - - if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) { - throw new IllegalArgumentException(tmpBaseDir + " is not a directory"); - } - if (!Files.isDirectory(outputDir)) { - throw new IllegalArgumentException(outputDir + " is not a directory"); - } - } - - public Path getOutputDir() { - return outputDir; - } - - public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { - - LOGGER.trace("start plot: {}", plotSettings); - - final String tmpSubDir = uniqueDirectoryName(); - final Path tmpDir = tmpBaseDir.resolve(tmpSubDir); - try { - Files.createDirectories(tmpDir); - final List dataSeries = Collections.synchronizedList(new ArrayList<>()); - - final String query = plotSettings.getQuery(); - final List groupBy = plotSettings.getGroupBy(); - final int height = plotSettings.getHeight(); - final int width = plotSettings.getWidth(); - final DateTimeRange dateRange = plotSettings.dateRange(); - final OffsetDateTime dateFrom = dateRange.getStart(); - final OffsetDateTime dateTo = dateRange.getEnd(); - - final Result result = db.get(new Query(query, dateRange), groupBy); - - final long start = System.nanoTime(); - final AtomicInteger idCounter = new AtomicInteger(0); - result.getGroups().stream().parallel().forEach(groupResult -> { - try { - final CsvSummary csvSummary = true - ? toCsvDeduplicated(groupResult, tmpDir, dateFrom, dateTo, plotSettings) - :toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings); - - final int id = idCounter.incrementAndGet(); - final String title = title(groupResult.getGroupedBy(), csvSummary); - final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary, - GnuplotLineType.Points); - if (dataSerie.getValues() > 0) { - dataSeries.add(dataSerie); - } - } catch (final Exception e) { - throw new IllegalStateException(e); - } - }); - METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); - - if (dataSeries.isEmpty()) { - throw new NoDataPointsException(); - } - - final Limit limitBy = plotSettings.getLimitBy(); - final int limit = plotSettings.getLimit(); - DataSeries.sortAndLimit(dataSeries, limitBy, limit); - DataSeries.setColors(dataSeries); - - final Path outputFile = Files.createTempFile(outputDir, "", ".png"); - { - final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); - final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile); - gnuplotSettings.setHeight(height); - gnuplotSettings.setWidth(width); - defineXAxis(gnuplotSettings, plotSettings.dateRange()); - - gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); - gnuplotSettings.setAggregate(plotSettings.getAggregate()); - defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(), - plotSettings.getYRangeUnit()); - gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); - gnuplot.plot(gnuplotSettings, dataSeries); - } - - final Path thumbnail; - if (plotSettings.isGenerateThumbnail()) { - thumbnail = Files.createTempFile(outputDir, "", ".png"); - final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); - final GnuplotSettings gnuplotSettings = new GnuplotSettings(thumbnail); - gnuplotSettings.setHeight(plotSettings.getThumbnailMaxHeight()); - gnuplotSettings.setWidth(plotSettings.getThumbnailMaxWidth()); - defineXAxis(gnuplotSettings, plotSettings.dateRange()); - - gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale()); - gnuplotSettings.setAggregate(plotSettings.getAggregate()); - defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(), - plotSettings.getYRangeUnit()); - gnuplotSettings.setKeyOutside(false); - gnuplotSettings.renderLabels(false); - gnuplot.plot(gnuplotSettings, dataSeries); - } else { - thumbnail = null; - } - - return new PlotResult(outputFile, dataSeries, thumbnail); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Plotting was interrupted."); - } catch (final IOException e) { - throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); - } finally { - FileUtils.delete(tmpDir); - LOGGER.trace("done plot"); - } - } - - private void defineYRange(final GnuplotSettings gnuplotSettings, final int yRangeMin, final int yRangeMax, - final TimeRangeUnitInternal yRangeUnit) { - - if (yRangeUnit != TimeRangeUnitInternal.AUTOMATIC) { - final int min = yRangeUnit.toMilliSeconds(yRangeMin); - final int max = yRangeUnit.toMilliSeconds(yRangeMax); - gnuplotSettings.setYRange(min, max); - } - } - - private void defineXAxis(final GnuplotSettings gnuplotSettings, final DateTimeRange dateTimeRange) { - - final OffsetDateTime minDate = dateTimeRange.getStart(); - final OffsetDateTime maxDate = dateTimeRange.getEnd(); - String formatX; - int rotateX; - String formattedMinDate; - String formattedMaxDate; - if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) { - formatX = "%Y-%m-%d"; - rotateX = 0; - } else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) { - formatX = "%Y-%m-%d\\n%H:%M:%S"; - rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); - } else { - formatX = "%Y-%m-%d\\n%H:%M:%.3S"; - rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel(); - } - formattedMinDate = String.valueOf(minDate.toEpochSecond()); - formattedMaxDate = String.valueOf(maxDate.toEpochSecond()); - - gnuplotSettings.getxAxisSettings().setFormatX(formatX); - gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX); - gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate); - gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate); - } - - private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom, - final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException { - - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - final long start = System.nanoTime(); - final Stream timeValueStream = groupResult.asStream(); - final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); - final long toEpochMilli = dateTo.toInstant().toEpochMilli(); - final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); - - final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0 - : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin()); - final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE - : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax()); - - final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli, - toEpochMilli); - - int count = 0; // number of values in the x-axis range (used to compute stats) - int plottedValues = 0; - long statsMaxValue = 0; - double statsCurrentAverage = 0.0; - long ignoredValues = 0; - final int separator = ','; - final int newline = '\n'; - final StringBuilder formattedDateBuilder = new StringBuilder(); - try (final Writer output = new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII)); - final Formatter formatter = new Formatter(formattedDateBuilder);) { - - final Iterator it = timeValueStream.iterator(); - while (it.hasNext()) { - final LongList entry = it.next(); - - for (int i = 0; i < entry.size(); i += 2) { - - final long epochMilli = entry.get(i); - if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) { - ignoredValues++; - continue; - } - - final long value = entry.get(i + 1); - - aggregator.addValue(epochMilli, value); - - // compute stats - count++; - statsMaxValue = Math.max(statsMaxValue, value); - - // compute average (important to do this after 'count' has been incremented) - statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count; - - // check if value is in the selected y-range - if (value < minValue || value > maxValue) { - ignoredValues++; - continue; - } - - final String stringValue = LongUtils.longToString(value); - final String formattedDate; - - if (useMillis) { - formattedDateBuilder.delete(0, formattedDateBuilder.length()); - formatter.format("%.3f", epochMilli / 1000.0); - formattedDate = formattedDateBuilder.toString(); - } else { - formattedDate = String.valueOf(epochMilli / 1000); - } - - output.write(formattedDate); - output.write(separator); - output.write(stringValue); - output.write(newline); - - plottedValues++; - } - } - } - - METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", - count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), - groupResult.getGroupedBy().asString(), dataFile); - return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage, - aggregator.getAggregatedData()); - - } - - private static CsvSummary toCsvDeduplicated(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom, - final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException { - - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - final long start = System.nanoTime(); - final Stream timeValueStream = groupResult.asStream(); - final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); - final long toEpochMilli = dateTo.toInstant().toEpochMilli(); - final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); - final long plotAreaWidthInPx = plotSettings.getWidth() - GnuplotSettings.GNUPLOT_LEFT_RIGHT_MARGIN; - final long plotAreaHeightInPx = plotSettings.getHeight() - GnuplotSettings.GNUPLOT_TOP_BOTTOM_MARGIN; - final long epochMillisPerPixel = Math.max(1, (toEpochMilli - fromEpochMilli) / plotAreaWidthInPx); - - final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0 - : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin()); - final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE - : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax()); - final long durationMillisPerPixel = plotSettings.getYAxisScale() == AxisScale.LINEAR - ? Math.max(1, (maxValue -minValue) / plotAreaHeightInPx) - : 1; - - final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli, - toEpochMilli); - - final Sparse2DLongArray matrix2d = new Sparse2DLongArray(); - int count = 0; // number of values in the x-axis range (used to compute stats) - int plottedValues = 0; - long statsMaxValue = 0; - double statsCurrentAverage = 0.0; - long ignoredValues = 0; - final int separator = ','; - final int newline = '\n'; - - final Iterator it = timeValueStream.iterator(); - while (it.hasNext()) { - final LongList entry = it.next(); - - for (int i = 0; i < entry.size(); i += 2) { - - final long epochMilli = entry.get(i); - if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) { - ignoredValues++; - continue; - } - - final long value = entry.get(i + 1); - - aggregator.addValue(epochMilli, value); - - // compute stats - count++; - statsMaxValue = Math.max(statsMaxValue, value); - - // compute average (important to do this after 'count' has been incremented) - statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count; - - // check if value is in the selected y-range - if (value < minValue || value > maxValue) { - ignoredValues++; - continue; - } - - final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel; - final long roundedValue = value - value % durationMillisPerPixel; - matrix2d.put(roundedEpochMilli, roundedValue, 1); - - plottedValues++; - } - } - - - long[] actualValuesWritten = new long[1]; - final StringBuilder formattedDateBuilder = new StringBuilder(); - try (final LambdaFriendlyWriter output = new LambdaFriendlyWriter(new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1))); - final Formatter formatter = new Formatter(formattedDateBuilder);) { - - matrix2d.forEach((epochMilli, value, __)-> { - - final String stringValue = LongUtils.longToString(value); - final String formattedDate; - - if (useMillis) { - formattedDateBuilder.delete(0, formattedDateBuilder.length()); - formatter.format("%.3f", epochMilli / 1000.0); - formattedDate = formattedDateBuilder.toString(); - } else { - formattedDate = String.valueOf(epochMilli / 1000); - } - - output.write(formattedDate); - output.write(separator); - output.write(stringValue); - output.write(newline); - actualValuesWritten[0]++; - }); - } - - METRICS_LOGGER.debug("wrote {} (actual: {} factor: {}%) values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", - actualValuesWritten[0], count, (double)count/(actualValuesWritten[0]), (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), - groupResult.getGroupedBy().asString(), dataFile); - return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage, - aggregator.getAggregatedData()); - - } - - static String uniqueDirectoryName() { - return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_" - + UUID.randomUUID().toString(); - } - - static String title(final Tags tags, final CsvSummary csvSummary) { - - final StringBuilder result = new StringBuilder(); - - final int values = csvSummary.getValues(); - final int plottedValues = csvSummary.getPlottedValues(); - - if (tags.isEmpty()) { - result.append(DEFAULT_GROUP); - } else { - tags.forEach((k, v) -> { - if (result.length() > 0) { - result.append(" / "); - } - result.append(v); - }); - } - - result.append(" ("); - if (plottedValues != values) { - result.append(String.format("%,d / %,d", plottedValues, values)); - } else { - result.append(String.format("%,d", values)); - } - result.append(")"); - - return result.toString(); - - } -}