merge ScatterPlot into Plotter
This commit is contained in:
@@ -1,40 +1,345 @@
|
||||
package org.lucares.recommind.logs;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Formatter;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.lucares.collections.LongList;
|
||||
import org.lucares.collections.Sparse2DLongArray;
|
||||
import org.lucares.pdb.api.DateTimeRange;
|
||||
import org.lucares.pdb.api.GroupResult;
|
||||
import org.lucares.pdb.api.Query;
|
||||
import org.lucares.pdb.api.Result;
|
||||
import org.lucares.pdb.api.Tags;
|
||||
import org.lucares.pdb.plot.api.AxisScale;
|
||||
import org.lucares.pdb.plot.api.CustomAggregator;
|
||||
import org.lucares.pdb.plot.api.Limit;
|
||||
import org.lucares.pdb.plot.api.PlotSettings;
|
||||
import org.lucares.pdb.plot.api.TimeRangeUnitInternal;
|
||||
import org.lucares.performance.db.PerformanceDb;
|
||||
import org.lucares.utils.file.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class Plotter {
|
||||
|
||||
private final PerformanceDb db;
|
||||
private final Path tmpBaseDir;
|
||||
private final Path outputDir;
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class);
|
||||
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter");
|
||||
|
||||
public Plotter(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
|
||||
this.db = db;
|
||||
this.tmpBaseDir = tmpBaseDir;
|
||||
this.outputDir = outputDir;
|
||||
static final String DEFAULT_GROUP = "<none>";
|
||||
|
||||
if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IllegalArgumentException(tmpBaseDir + " is not a directory");
|
||||
}
|
||||
if (!Files.isDirectory(outputDir)) {
|
||||
throw new IllegalArgumentException(outputDir + " is not a directory");
|
||||
}
|
||||
}
|
||||
private final PerformanceDb db;
|
||||
private final Path tmpBaseDir;
|
||||
private final Path outputDir;
|
||||
|
||||
public Path getOutputDir() {
|
||||
return outputDir;
|
||||
}
|
||||
public Plotter(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
|
||||
this.db = db;
|
||||
this.tmpBaseDir = tmpBaseDir;
|
||||
this.outputDir = outputDir;
|
||||
|
||||
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
|
||||
if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IllegalArgumentException(tmpBaseDir + " is not a directory");
|
||||
}
|
||||
if (!Files.isDirectory(outputDir)) {
|
||||
throw new IllegalArgumentException(outputDir + " is not a directory");
|
||||
}
|
||||
}
|
||||
|
||||
final ScatterPlot plotter = new ScatterPlot(db, tmpBaseDir, outputDir);
|
||||
public Path getOutputDir() {
|
||||
return outputDir;
|
||||
}
|
||||
|
||||
return plotter.plot(plotSettings);
|
||||
}
|
||||
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
|
||||
|
||||
LOGGER.trace("start plot: {}", plotSettings);
|
||||
|
||||
final String tmpSubDir = uniqueDirectoryName();
|
||||
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
|
||||
try {
|
||||
Files.createDirectories(tmpDir);
|
||||
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
final String query = plotSettings.getQuery();
|
||||
final List<String> groupBy = plotSettings.getGroupBy();
|
||||
final int height = plotSettings.getHeight();
|
||||
final int width = plotSettings.getWidth();
|
||||
final DateTimeRange dateRange = plotSettings.dateRange();
|
||||
final OffsetDateTime dateFrom = dateRange.getStart();
|
||||
final OffsetDateTime dateTo = dateRange.getEnd();
|
||||
|
||||
final Result result = db.get(new Query(query, dateRange), groupBy);
|
||||
|
||||
final long start = System.nanoTime();
|
||||
final AtomicInteger idCounter = new AtomicInteger(0);
|
||||
result.getGroups().stream().parallel().forEach(groupResult -> {
|
||||
try {
|
||||
final CsvSummary csvSummary = toCsvDeduplicated(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
|
||||
|
||||
final int id = idCounter.incrementAndGet();
|
||||
final String title = title(groupResult.getGroupedBy(), csvSummary);
|
||||
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.Points);
|
||||
if (dataSerie.getValues() > 0) {
|
||||
dataSeries.add(dataSerie);
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
});
|
||||
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
|
||||
|
||||
if (dataSeries.isEmpty()) {
|
||||
throw new NoDataPointsException();
|
||||
}
|
||||
|
||||
final Limit limitBy = plotSettings.getLimitBy();
|
||||
final int limit = plotSettings.getLimit();
|
||||
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
|
||||
DataSeries.setColors(dataSeries);
|
||||
|
||||
final Path outputFile = Files.createTempFile(outputDir, "", ".png");
|
||||
{
|
||||
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
|
||||
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
|
||||
gnuplotSettings.setHeight(height);
|
||||
gnuplotSettings.setWidth(width);
|
||||
defineXAxis(gnuplotSettings, plotSettings.dateRange());
|
||||
|
||||
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
|
||||
gnuplotSettings.setAggregate(plotSettings.getAggregate());
|
||||
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
|
||||
plotSettings.getYRangeUnit());
|
||||
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
|
||||
gnuplot.plot(gnuplotSettings, dataSeries);
|
||||
}
|
||||
|
||||
final Path thumbnail;
|
||||
if (plotSettings.isGenerateThumbnail()) {
|
||||
thumbnail = Files.createTempFile(outputDir, "", ".png");
|
||||
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
|
||||
final GnuplotSettings gnuplotSettings = new GnuplotSettings(thumbnail);
|
||||
gnuplotSettings.setHeight(plotSettings.getThumbnailMaxHeight());
|
||||
gnuplotSettings.setWidth(plotSettings.getThumbnailMaxWidth());
|
||||
defineXAxis(gnuplotSettings, plotSettings.dateRange());
|
||||
|
||||
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
|
||||
gnuplotSettings.setAggregate(plotSettings.getAggregate());
|
||||
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
|
||||
plotSettings.getYRangeUnit());
|
||||
gnuplotSettings.setKeyOutside(false);
|
||||
gnuplotSettings.renderLabels(false);
|
||||
gnuplot.plot(gnuplotSettings, dataSeries);
|
||||
} else {
|
||||
thumbnail = null;
|
||||
}
|
||||
|
||||
return new PlotResult(outputFile, dataSeries, thumbnail);
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("Plotting was interrupted.");
|
||||
} catch (final IOException e) {
|
||||
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
|
||||
} finally {
|
||||
FileUtils.delete(tmpDir);
|
||||
LOGGER.trace("done plot");
|
||||
}
|
||||
}
|
||||
|
||||
private void defineYRange(final GnuplotSettings gnuplotSettings, final int yRangeMin, final int yRangeMax,
|
||||
final TimeRangeUnitInternal yRangeUnit) {
|
||||
|
||||
if (yRangeUnit != TimeRangeUnitInternal.AUTOMATIC) {
|
||||
final int min = yRangeUnit.toMilliSeconds(yRangeMin);
|
||||
final int max = yRangeUnit.toMilliSeconds(yRangeMax);
|
||||
gnuplotSettings.setYRange(min, max);
|
||||
}
|
||||
}
|
||||
|
||||
private void defineXAxis(final GnuplotSettings gnuplotSettings, final DateTimeRange dateTimeRange) {
|
||||
|
||||
final OffsetDateTime minDate = dateTimeRange.getStart();
|
||||
final OffsetDateTime maxDate = dateTimeRange.getEnd();
|
||||
String formatX;
|
||||
int rotateX;
|
||||
String formattedMinDate;
|
||||
String formattedMaxDate;
|
||||
if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) {
|
||||
formatX = "%Y-%m-%d";
|
||||
rotateX = 0;
|
||||
} else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) {
|
||||
formatX = "%Y-%m-%d\\n%H:%M:%S";
|
||||
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
|
||||
} else {
|
||||
formatX = "%Y-%m-%d\\n%H:%M:%.3S";
|
||||
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
|
||||
}
|
||||
formattedMinDate = String.valueOf(minDate.toEpochSecond());
|
||||
formattedMaxDate = String.valueOf(maxDate.toEpochSecond());
|
||||
|
||||
gnuplotSettings.getxAxisSettings().setFormatX(formatX);
|
||||
gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX);
|
||||
gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate);
|
||||
gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate);
|
||||
}
|
||||
|
||||
private static CsvSummary toCsvDeduplicated(final GroupResult groupResult, final Path tmpDir,
|
||||
final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException {
|
||||
|
||||
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
|
||||
final long start = System.nanoTime();
|
||||
final Stream<LongList> timeValueStream = groupResult.asStream();
|
||||
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
|
||||
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
|
||||
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
|
||||
final long plotAreaWidthInPx = plotSettings.getWidth() - GnuplotSettings.GNUPLOT_LEFT_RIGHT_MARGIN;
|
||||
final long plotAreaHeightInPx = plotSettings.getHeight() - GnuplotSettings.GNUPLOT_TOP_BOTTOM_MARGIN;
|
||||
final long epochMillisPerPixel = Math.max(1, (toEpochMilli - fromEpochMilli) / plotAreaWidthInPx);
|
||||
|
||||
final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin());
|
||||
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
|
||||
final long durationMillisPerPixel = plotSettings.getYAxisScale() == AxisScale.LINEAR
|
||||
? Math.max(1, (maxValue - minValue) / plotAreaHeightInPx)
|
||||
: 1;
|
||||
|
||||
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli,
|
||||
toEpochMilli);
|
||||
|
||||
final Sparse2DLongArray matrix2d = new Sparse2DLongArray();
|
||||
int count = 0; // number of values in the x-axis range (used to compute stats)
|
||||
int plottedValues = 0;
|
||||
long statsMaxValue = 0;
|
||||
double statsCurrentAverage = 0.0;
|
||||
long ignoredValues = 0;
|
||||
final int separator = ',';
|
||||
final int newline = '\n';
|
||||
|
||||
final Iterator<LongList> it = timeValueStream.iterator();
|
||||
while (it.hasNext()) {
|
||||
final LongList entry = it.next();
|
||||
|
||||
for (int i = 0; i < entry.size(); i += 2) {
|
||||
|
||||
final long epochMilli = entry.get(i);
|
||||
if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final long value = entry.get(i + 1);
|
||||
|
||||
aggregator.addValue(epochMilli, value);
|
||||
|
||||
// compute stats
|
||||
count++;
|
||||
statsMaxValue = Math.max(statsMaxValue, value);
|
||||
|
||||
// compute average (important to do this after 'count' has been incremented)
|
||||
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
|
||||
|
||||
// check if value is in the selected y-range
|
||||
if (value < minValue || value > maxValue) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel;
|
||||
final long roundedValue = value - value % durationMillisPerPixel;
|
||||
matrix2d.put(roundedEpochMilli, roundedValue, 1);
|
||||
|
||||
plottedValues++;
|
||||
}
|
||||
}
|
||||
|
||||
long[] actualValuesWritten = new long[1];
|
||||
final StringBuilder formattedDateBuilder = new StringBuilder();
|
||||
try (
|
||||
final LambdaFriendlyWriter output = new LambdaFriendlyWriter(
|
||||
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1)));
|
||||
final Formatter formatter = new Formatter(formattedDateBuilder);) {
|
||||
|
||||
matrix2d.forEach((epochMilli, value, __) -> {
|
||||
|
||||
final String stringValue = LongUtils.longToString(value);
|
||||
final String formattedDate;
|
||||
|
||||
if (useMillis) {
|
||||
formattedDateBuilder.delete(0, formattedDateBuilder.length());
|
||||
formatter.format("%.3f", epochMilli / 1000.0);
|
||||
formattedDate = formattedDateBuilder.toString();
|
||||
} else {
|
||||
formattedDate = String.valueOf(epochMilli / 1000);
|
||||
}
|
||||
|
||||
output.write(formattedDate);
|
||||
output.write(separator);
|
||||
output.write(stringValue);
|
||||
output.write(newline);
|
||||
actualValuesWritten[0]++;
|
||||
});
|
||||
}
|
||||
|
||||
METRICS_LOGGER.debug(
|
||||
"wrote {} (actual: {} factor: {}%) values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
|
||||
actualValuesWritten[0], count, (double) count / (actualValuesWritten[0]),
|
||||
(System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
|
||||
groupResult.getGroupedBy().asString(), dataFile);
|
||||
return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
|
||||
aggregator.getAggregatedData());
|
||||
|
||||
}
|
||||
|
||||
static String uniqueDirectoryName() {
|
||||
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
|
||||
+ UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
static String title(final Tags tags, final CsvSummary csvSummary) {
|
||||
|
||||
final StringBuilder result = new StringBuilder();
|
||||
|
||||
final int values = csvSummary.getValues();
|
||||
final int plottedValues = csvSummary.getPlottedValues();
|
||||
|
||||
if (tags.isEmpty()) {
|
||||
result.append(DEFAULT_GROUP);
|
||||
} else {
|
||||
tags.forEach((k, v) -> {
|
||||
if (result.length() > 0) {
|
||||
result.append(" / ");
|
||||
}
|
||||
result.append(v);
|
||||
});
|
||||
}
|
||||
|
||||
result.append(" (");
|
||||
if (plottedValues != values) {
|
||||
result.append(String.format("%,d / %,d", plottedValues, values));
|
||||
} else {
|
||||
result.append(String.format("%,d", values));
|
||||
}
|
||||
result.append(")");
|
||||
|
||||
return result.toString();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,435 +0,0 @@
|
||||
package org.lucares.recommind.logs;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.LinkOption;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Formatter;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.lucares.collections.LongList;
|
||||
import org.lucares.collections.Sparse2DLongArray;
|
||||
import org.lucares.pdb.api.DateTimeRange;
|
||||
import org.lucares.pdb.api.GroupResult;
|
||||
import org.lucares.pdb.api.Query;
|
||||
import org.lucares.pdb.api.Result;
|
||||
import org.lucares.pdb.api.Tags;
|
||||
import org.lucares.pdb.plot.api.AxisScale;
|
||||
import org.lucares.pdb.plot.api.CustomAggregator;
|
||||
import org.lucares.pdb.plot.api.Limit;
|
||||
import org.lucares.pdb.plot.api.PlotSettings;
|
||||
import org.lucares.pdb.plot.api.TimeRangeUnitInternal;
|
||||
import org.lucares.performance.db.PerformanceDb;
|
||||
import org.lucares.utils.file.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class ScatterPlot {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class);
|
||||
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter");
|
||||
|
||||
static final String DEFAULT_GROUP = "<none>";
|
||||
|
||||
private final PerformanceDb db;
|
||||
private final Path tmpBaseDir;
|
||||
private final Path outputDir;
|
||||
|
||||
public ScatterPlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
|
||||
this.db = db;
|
||||
this.tmpBaseDir = tmpBaseDir;
|
||||
this.outputDir = outputDir;
|
||||
|
||||
if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) {
|
||||
throw new IllegalArgumentException(tmpBaseDir + " is not a directory");
|
||||
}
|
||||
if (!Files.isDirectory(outputDir)) {
|
||||
throw new IllegalArgumentException(outputDir + " is not a directory");
|
||||
}
|
||||
}
|
||||
|
||||
public Path getOutputDir() {
|
||||
return outputDir;
|
||||
}
|
||||
|
||||
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
|
||||
|
||||
LOGGER.trace("start plot: {}", plotSettings);
|
||||
|
||||
final String tmpSubDir = uniqueDirectoryName();
|
||||
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
|
||||
try {
|
||||
Files.createDirectories(tmpDir);
|
||||
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
final String query = plotSettings.getQuery();
|
||||
final List<String> groupBy = plotSettings.getGroupBy();
|
||||
final int height = plotSettings.getHeight();
|
||||
final int width = plotSettings.getWidth();
|
||||
final DateTimeRange dateRange = plotSettings.dateRange();
|
||||
final OffsetDateTime dateFrom = dateRange.getStart();
|
||||
final OffsetDateTime dateTo = dateRange.getEnd();
|
||||
|
||||
final Result result = db.get(new Query(query, dateRange), groupBy);
|
||||
|
||||
final long start = System.nanoTime();
|
||||
final AtomicInteger idCounter = new AtomicInteger(0);
|
||||
result.getGroups().stream().parallel().forEach(groupResult -> {
|
||||
try {
|
||||
final CsvSummary csvSummary = true
|
||||
? toCsvDeduplicated(groupResult, tmpDir, dateFrom, dateTo, plotSettings)
|
||||
:toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
|
||||
|
||||
final int id = idCounter.incrementAndGet();
|
||||
final String title = title(groupResult.getGroupedBy(), csvSummary);
|
||||
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary,
|
||||
GnuplotLineType.Points);
|
||||
if (dataSerie.getValues() > 0) {
|
||||
dataSeries.add(dataSerie);
|
||||
}
|
||||
} catch (final Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
});
|
||||
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
|
||||
|
||||
if (dataSeries.isEmpty()) {
|
||||
throw new NoDataPointsException();
|
||||
}
|
||||
|
||||
final Limit limitBy = plotSettings.getLimitBy();
|
||||
final int limit = plotSettings.getLimit();
|
||||
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
|
||||
DataSeries.setColors(dataSeries);
|
||||
|
||||
final Path outputFile = Files.createTempFile(outputDir, "", ".png");
|
||||
{
|
||||
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
|
||||
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
|
||||
gnuplotSettings.setHeight(height);
|
||||
gnuplotSettings.setWidth(width);
|
||||
defineXAxis(gnuplotSettings, plotSettings.dateRange());
|
||||
|
||||
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
|
||||
gnuplotSettings.setAggregate(plotSettings.getAggregate());
|
||||
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
|
||||
plotSettings.getYRangeUnit());
|
||||
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
|
||||
gnuplot.plot(gnuplotSettings, dataSeries);
|
||||
}
|
||||
|
||||
final Path thumbnail;
|
||||
if (plotSettings.isGenerateThumbnail()) {
|
||||
thumbnail = Files.createTempFile(outputDir, "", ".png");
|
||||
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
|
||||
final GnuplotSettings gnuplotSettings = new GnuplotSettings(thumbnail);
|
||||
gnuplotSettings.setHeight(plotSettings.getThumbnailMaxHeight());
|
||||
gnuplotSettings.setWidth(plotSettings.getThumbnailMaxWidth());
|
||||
defineXAxis(gnuplotSettings, plotSettings.dateRange());
|
||||
|
||||
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
|
||||
gnuplotSettings.setAggregate(plotSettings.getAggregate());
|
||||
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
|
||||
plotSettings.getYRangeUnit());
|
||||
gnuplotSettings.setKeyOutside(false);
|
||||
gnuplotSettings.renderLabels(false);
|
||||
gnuplot.plot(gnuplotSettings, dataSeries);
|
||||
} else {
|
||||
thumbnail = null;
|
||||
}
|
||||
|
||||
return new PlotResult(outputFile, dataSeries, thumbnail);
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("Plotting was interrupted.");
|
||||
} catch (final IOException e) {
|
||||
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
|
||||
} finally {
|
||||
FileUtils.delete(tmpDir);
|
||||
LOGGER.trace("done plot");
|
||||
}
|
||||
}
|
||||
|
||||
private void defineYRange(final GnuplotSettings gnuplotSettings, final int yRangeMin, final int yRangeMax,
|
||||
final TimeRangeUnitInternal yRangeUnit) {
|
||||
|
||||
if (yRangeUnit != TimeRangeUnitInternal.AUTOMATIC) {
|
||||
final int min = yRangeUnit.toMilliSeconds(yRangeMin);
|
||||
final int max = yRangeUnit.toMilliSeconds(yRangeMax);
|
||||
gnuplotSettings.setYRange(min, max);
|
||||
}
|
||||
}
|
||||
|
||||
private void defineXAxis(final GnuplotSettings gnuplotSettings, final DateTimeRange dateTimeRange) {
|
||||
|
||||
final OffsetDateTime minDate = dateTimeRange.getStart();
|
||||
final OffsetDateTime maxDate = dateTimeRange.getEnd();
|
||||
String formatX;
|
||||
int rotateX;
|
||||
String formattedMinDate;
|
||||
String formattedMaxDate;
|
||||
if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) {
|
||||
formatX = "%Y-%m-%d";
|
||||
rotateX = 0;
|
||||
} else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) {
|
||||
formatX = "%Y-%m-%d\\n%H:%M:%S";
|
||||
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
|
||||
} else {
|
||||
formatX = "%Y-%m-%d\\n%H:%M:%.3S";
|
||||
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
|
||||
}
|
||||
formattedMinDate = String.valueOf(minDate.toEpochSecond());
|
||||
formattedMaxDate = String.valueOf(maxDate.toEpochSecond());
|
||||
|
||||
gnuplotSettings.getxAxisSettings().setFormatX(formatX);
|
||||
gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX);
|
||||
gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate);
|
||||
gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate);
|
||||
}
|
||||
|
||||
private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom,
|
||||
final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException {
|
||||
|
||||
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
|
||||
final long start = System.nanoTime();
|
||||
final Stream<LongList> timeValueStream = groupResult.asStream();
|
||||
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
|
||||
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
|
||||
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
|
||||
|
||||
final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin());
|
||||
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
|
||||
|
||||
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli,
|
||||
toEpochMilli);
|
||||
|
||||
int count = 0; // number of values in the x-axis range (used to compute stats)
|
||||
int plottedValues = 0;
|
||||
long statsMaxValue = 0;
|
||||
double statsCurrentAverage = 0.0;
|
||||
long ignoredValues = 0;
|
||||
final int separator = ',';
|
||||
final int newline = '\n';
|
||||
final StringBuilder formattedDateBuilder = new StringBuilder();
|
||||
try (final Writer output = new BufferedWriter(
|
||||
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));
|
||||
final Formatter formatter = new Formatter(formattedDateBuilder);) {
|
||||
|
||||
final Iterator<LongList> it = timeValueStream.iterator();
|
||||
while (it.hasNext()) {
|
||||
final LongList entry = it.next();
|
||||
|
||||
for (int i = 0; i < entry.size(); i += 2) {
|
||||
|
||||
final long epochMilli = entry.get(i);
|
||||
if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final long value = entry.get(i + 1);
|
||||
|
||||
aggregator.addValue(epochMilli, value);
|
||||
|
||||
// compute stats
|
||||
count++;
|
||||
statsMaxValue = Math.max(statsMaxValue, value);
|
||||
|
||||
// compute average (important to do this after 'count' has been incremented)
|
||||
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
|
||||
|
||||
// check if value is in the selected y-range
|
||||
if (value < minValue || value > maxValue) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final String stringValue = LongUtils.longToString(value);
|
||||
final String formattedDate;
|
||||
|
||||
if (useMillis) {
|
||||
formattedDateBuilder.delete(0, formattedDateBuilder.length());
|
||||
formatter.format("%.3f", epochMilli / 1000.0);
|
||||
formattedDate = formattedDateBuilder.toString();
|
||||
} else {
|
||||
formattedDate = String.valueOf(epochMilli / 1000);
|
||||
}
|
||||
|
||||
output.write(formattedDate);
|
||||
output.write(separator);
|
||||
output.write(stringValue);
|
||||
output.write(newline);
|
||||
|
||||
plottedValues++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
|
||||
count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
|
||||
groupResult.getGroupedBy().asString(), dataFile);
|
||||
return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
|
||||
aggregator.getAggregatedData());
|
||||
|
||||
}
|
||||
|
||||
private static CsvSummary toCsvDeduplicated(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom,
|
||||
final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException {
|
||||
|
||||
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
|
||||
final long start = System.nanoTime();
|
||||
final Stream<LongList> timeValueStream = groupResult.asStream();
|
||||
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
|
||||
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
|
||||
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
|
||||
final long plotAreaWidthInPx = plotSettings.getWidth() - GnuplotSettings.GNUPLOT_LEFT_RIGHT_MARGIN;
|
||||
final long plotAreaHeightInPx = plotSettings.getHeight() - GnuplotSettings.GNUPLOT_TOP_BOTTOM_MARGIN;
|
||||
final long epochMillisPerPixel = Math.max(1, (toEpochMilli - fromEpochMilli) / plotAreaWidthInPx);
|
||||
|
||||
final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin());
|
||||
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
|
||||
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
|
||||
final long durationMillisPerPixel = plotSettings.getYAxisScale() == AxisScale.LINEAR
|
||||
? Math.max(1, (maxValue -minValue) / plotAreaHeightInPx)
|
||||
: 1;
|
||||
|
||||
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli,
|
||||
toEpochMilli);
|
||||
|
||||
final Sparse2DLongArray matrix2d = new Sparse2DLongArray();
|
||||
int count = 0; // number of values in the x-axis range (used to compute stats)
|
||||
int plottedValues = 0;
|
||||
long statsMaxValue = 0;
|
||||
double statsCurrentAverage = 0.0;
|
||||
long ignoredValues = 0;
|
||||
final int separator = ',';
|
||||
final int newline = '\n';
|
||||
|
||||
final Iterator<LongList> it = timeValueStream.iterator();
|
||||
while (it.hasNext()) {
|
||||
final LongList entry = it.next();
|
||||
|
||||
for (int i = 0; i < entry.size(); i += 2) {
|
||||
|
||||
final long epochMilli = entry.get(i);
|
||||
if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final long value = entry.get(i + 1);
|
||||
|
||||
aggregator.addValue(epochMilli, value);
|
||||
|
||||
// compute stats
|
||||
count++;
|
||||
statsMaxValue = Math.max(statsMaxValue, value);
|
||||
|
||||
// compute average (important to do this after 'count' has been incremented)
|
||||
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
|
||||
|
||||
// check if value is in the selected y-range
|
||||
if (value < minValue || value > maxValue) {
|
||||
ignoredValues++;
|
||||
continue;
|
||||
}
|
||||
|
||||
final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel;
|
||||
final long roundedValue = value - value % durationMillisPerPixel;
|
||||
matrix2d.put(roundedEpochMilli, roundedValue, 1);
|
||||
|
||||
plottedValues++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
long[] actualValuesWritten = new long[1];
|
||||
final StringBuilder formattedDateBuilder = new StringBuilder();
|
||||
try (final LambdaFriendlyWriter output = new LambdaFriendlyWriter(new BufferedWriter(
|
||||
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1)));
|
||||
final Formatter formatter = new Formatter(formattedDateBuilder);) {
|
||||
|
||||
matrix2d.forEach((epochMilli, value, __)-> {
|
||||
|
||||
final String stringValue = LongUtils.longToString(value);
|
||||
final String formattedDate;
|
||||
|
||||
if (useMillis) {
|
||||
formattedDateBuilder.delete(0, formattedDateBuilder.length());
|
||||
formatter.format("%.3f", epochMilli / 1000.0);
|
||||
formattedDate = formattedDateBuilder.toString();
|
||||
} else {
|
||||
formattedDate = String.valueOf(epochMilli / 1000);
|
||||
}
|
||||
|
||||
output.write(formattedDate);
|
||||
output.write(separator);
|
||||
output.write(stringValue);
|
||||
output.write(newline);
|
||||
actualValuesWritten[0]++;
|
||||
});
|
||||
}
|
||||
|
||||
METRICS_LOGGER.debug("wrote {} (actual: {} factor: {}%) values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
|
||||
actualValuesWritten[0], count, (double)count/(actualValuesWritten[0]), (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
|
||||
groupResult.getGroupedBy().asString(), dataFile);
|
||||
return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
|
||||
aggregator.getAggregatedData());
|
||||
|
||||
}
|
||||
|
||||
static String uniqueDirectoryName() {
|
||||
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
|
||||
+ UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
static String title(final Tags tags, final CsvSummary csvSummary) {
|
||||
|
||||
final StringBuilder result = new StringBuilder();
|
||||
|
||||
final int values = csvSummary.getValues();
|
||||
final int plottedValues = csvSummary.getPlottedValues();
|
||||
|
||||
if (tags.isEmpty()) {
|
||||
result.append(DEFAULT_GROUP);
|
||||
} else {
|
||||
tags.forEach((k, v) -> {
|
||||
if (result.length() > 0) {
|
||||
result.append(" / ");
|
||||
}
|
||||
result.append(v);
|
||||
});
|
||||
}
|
||||
|
||||
result.append(" (");
|
||||
if (plottedValues != values) {
|
||||
result.append(String.format("%,d / %,d", plottedValues, values));
|
||||
} else {
|
||||
result.append(String.format("%,d", values));
|
||||
}
|
||||
result.append(")");
|
||||
|
||||
return result.toString();
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user