andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

move scatter plot creation into an AggregateHandler

Browse Source
This commit is contained in:
Andreas Huber
2019-10-20 08:11:09 +02:00
parent 7c61686808
commit b7c4fe4c1f
19 changed files with 186 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java
View File

@@ -1,21 +1,17 @@
package org.lucares.recommind.logs;
import java.io.File;
import org.lucares.pdb.plot.api.AggregatedData;
class CsvSummary {
private final int values;
private final long maxValue;
private final File dataFile;
private final AggregatedData aggregatedData;
private final double statsAverage;
private final int plottedValues;
public CsvSummary(final File dataFile, final int values, final int plottedValues, final long maxValue,
public CsvSummary(final int values, final int plottedValues, final long maxValue,
final double statsAverage, final AggregatedData aggregatedData) {
super();
this.dataFile = dataFile;
this.values = values;
this.plottedValues = plottedValues;
this.maxValue = maxValue;
@@ -23,9 +19,6 @@ class CsvSummary {
this.aggregatedData = aggregatedData;
}
public File getDataFile() {
return dataFile;
}
/**
* Total number of values in the selected date range.

2
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -43,8 +43,6 @@ public interface DataSeries {
public AggregatedData getAggregatedData();
public String getGnuplotPlotDefinition();
public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
final Map<String, Integer> result = new LinkedHashMap<>();

23
pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java
View File

@@ -1,7 +1,5 @@
package org.lucares.recommind.logs;
import java.io.File;
import org.lucares.pdb.plot.api.AggregatedData;
public class FileBackedDataSeries implements DataSeries {
@@ -12,16 +10,13 @@ public class FileBackedDataSeries implements DataSeries {
private final int id;
private final GnuplotLineType linetype;
private LineStyle style;
public FileBackedDataSeries(final int id, final String title, final CsvSummary csvSummary,
final GnuplotLineType linetype) {
public FileBackedDataSeries(final int id, final String title, final CsvSummary csvSummary
) {
this.id = id;
this.title = title;
this.csvSummary = csvSummary;
this.linetype = linetype;
}
@Override
@@ -44,9 +39,6 @@ public class FileBackedDataSeries implements DataSeries {
return style;
}
public File getDataFile() {
return csvSummary.getDataFile();
}
@Override
public String getTitle() {
@@ -77,15 +69,4 @@ public class FileBackedDataSeries implements DataSeries {
public AggregatedData getAggregatedData() {
return csvSummary.getAggregatedData();
}
@Override
public String getGnuplotPlotDefinition() {
return String.format("'%s' using 1:2 title '%s' with %s %s, \\", //
getDataFile(), //
getTitle(), //
linetype, // line or points
style//
);
}
}

5
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -60,6 +60,7 @@ public class GnuplotFileGenerator {
appendfln(result, "set grid");
appendfln(result, "set output \"%s\"", settings.getOutput().toAbsolutePath().toString().replace("\\", "/"));
// TODO remove marker lines?
// marker lines that show which area will be zoomed
appendfln(result, "set arrow from graph 0.25,0 rto graph 0,1 lc rgb \"#EEEEEE\" nohead");
appendfln(result, "set arrow from graph 0.75,0 rto graph 0,1 lc rgb \"#EEEEEE\" nohead");
@@ -103,9 +104,7 @@ public class GnuplotFileGenerator {
appendf(result, "plot ");
settings.getAggregate().addPlotsBeforeScatter(result, dataSeries);
for (final DataSeries dataSerie : dataSeries) {
appendfln(result, dataSerie.getGnuplotPlotDefinition());
}
settings.getAggregate().addPlotsAfterScatter(result, dataSeries);
return result.toString();

74
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -1,11 +1,6 @@
package org.lucares.recommind.logs;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
@@ -14,7 +9,6 @@ import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -23,13 +17,11 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import org.lucares.collections.LongList;
import org.lucares.collections.Sparse2DLongArray;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Query;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
@@ -95,7 +87,7 @@ public class Plotter {
final int id = idCounter.incrementAndGet();
final String title = title(groupResult.getGroupedBy(), csvSummary);
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.Points);
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
@@ -202,35 +194,25 @@ public class Plotter {
private static CsvSummary toCsvDeduplicated(final GroupResult groupResult, final Path tmpDir,
final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings) throws IOException {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
final long start = System.nanoTime();
final Stream<LongList> timeValueStream = groupResult.asStream();
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final long plotAreaWidthInPx = plotSettings.getWidth() - GnuplotSettings.GNUPLOT_LEFT_RIGHT_MARGIN;
final long plotAreaHeightInPx = plotSettings.getHeight() - GnuplotSettings.GNUPLOT_TOP_BOTTOM_MARGIN;
final long epochMillisPerPixel = Math.max(1, (toEpochMilli - fromEpochMilli) / plotAreaWidthInPx);
final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin());
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
final long durationMillisPerPixel = plotSettings.getYAxisScale() == AxisScale.LINEAR
? Math.max(1, (maxValue - minValue) / plotAreaHeightInPx)
: 1;
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, fromEpochMilli,
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, plotSettings, fromEpochMilli,
toEpochMilli);
final Sparse2DLongArray matrix2d = new Sparse2DLongArray();
int count = 0; // number of values in the x-axis range (used to compute stats)
int plottedValues = 0;
long statsMaxValue = 0;
double statsCurrentAverage = 0.0;
long ignoredValues = 0;
final int separator = ',';
final int newline = '\n';
final Iterator<LongList> it = timeValueStream.iterator();
while (it.hasNext()) {
@@ -246,7 +228,7 @@ public class Plotter {
final long value = entry.get(i + 1);
aggregator.addValue(epochMilli, value);
// compute stats
count++;
@@ -256,53 +238,25 @@ public class Plotter {
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
// check if value is in the selected y-range
if (value < minValue || value > maxValue) {
boolean valueIsInYRange = value < minValue || value > maxValue;
if (valueIsInYRange) {
ignoredValues++;
continue;
}else {
plottedValues++;
}
final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel;
final long roundedValue = value - value % durationMillisPerPixel;
matrix2d.put(roundedEpochMilli, roundedValue, 1);
plottedValues++;
aggregator.addValue(valueIsInYRange, epochMilli, value);
}
}
long[] actualValuesWritten = new long[1];
final StringBuilder formattedDateBuilder = new StringBuilder();
try (
final LambdaFriendlyWriter output = new LambdaFriendlyWriter(
new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1)));
final Formatter formatter = new Formatter(formattedDateBuilder);) {
matrix2d.forEach((epochMilli, value, __) -> {
final String stringValue = LongUtils.longToString(value);
final String formattedDate;
if (useMillis) {
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
} else {
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(stringValue);
output.write(newline);
actualValuesWritten[0]++;
});
}
METRICS_LOGGER.debug(
"wrote {} (actual: {} factor: {}%) values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
actualValuesWritten[0], count, (double) count / (actualValuesWritten[0]),
"wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}",
plottedValues,
(System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
groupResult.getGroupedBy().asString(), dataFile);
return new CsvSummary(dataFile, count, plottedValues, statsMaxValue, statsCurrentAverage,
groupResult.getGroupedBy().asString());
return new CsvSummary( count, plottedValues, statsMaxValue, statsCurrentAverage,
aggregator.getAggregatedData());
}