andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

remove percentile plot

Browse Source 
Eventually we want to only support what is now called aggregate, but
not have to implement different plot types. So instead of supporting
percentile plots for dashboards I removed them. You can still get
percentile plots together with the scatter plot.
This commit is contained in:
Andreas Huber
2018-05-01 08:27:37 +02:00
parent 1d8f31808e
commit fc64c55ce7
10 changed files with 40 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java
View File

@@ -40,8 +40,6 @@ public class PlotSettings {
private boolean keyOutside;
private PlotType plotType;
private boolean generateThumbnail;
public String getQuery() {
@@ -185,8 +183,7 @@ public class PlotSettings {
return "PlotSettings [query=" + query + ", height=" + height + ", width=" + width + ", thumbnailMaxWidth="
+ thumbnailMaxWidth + ", thumbnailMaxHeight=" + thumbnailMaxHeight + ", groupBy=" + groupBy
+ ", limitBy=" + limitBy + ", limit=" + limit + ", dateFrom=" + dateFrom + ", dateRange=" + dateRange
+ ", yAxisScale=" + yAxisScale + ", aggregate=" + aggregate + ", keyOutside=" + keyOutside
+ ", plotType=" + plotType + "]";
+ ", yAxisScale=" + yAxisScale + ", aggregate=" + aggregate + ", keyOutside=" + keyOutside + "]";
}
public void setAggregate(final AggregateHandler aggregate) {
@@ -205,14 +202,6 @@ public class PlotSettings {
return keyOutside;
}
public void setPlotType(final PlotType plotType) {
this.plotType = plotType;
}
public PlotType getPlotType() {
return plotType;
}
public void setGenerateThumbnail(final boolean generateThumbnail) {
this.generateThumbnail = generateThumbnail;
}

5
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotType.java
View File

@@ -1,5 +0,0 @@
package org.lucares.pdb.plot.api;
public enum PlotType {
SCATTER, PERCENTILES
}

45
pdb-plotting/src/main/java/org/lucares/recommind/logs/ConcretePlotter.java
View File

@@ -1,45 +0,0 @@
package org.lucares.recommind.logs;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.UUID;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.PlotSettings;
public interface ConcretePlotter {
static final String DEFAULT_GROUP = "<none>";
PlotResult plot(PlotSettings plotSettings) throws InternalPlottingException;
static String uniqueDirectoryName() {
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
+ UUID.randomUUID().toString();
}
static String title(final Tags tags, final int values) {
final StringBuilder result = new StringBuilder();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append(" (");
result.append(String.format("%,d", values));
result.append(")");
return result.toString();
}
}

189
pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java
View File

@@ -1,189 +0,0 @@
package org.lucares.recommind.logs;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import org.lucares.collections.IntList;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PercentilePlot implements ConcretePlotter {
private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.percentile");
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
public PercentilePlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
this.db = db;
this.tmpBaseDir = tmpBaseDir;
this.outputDir = outputDir;
}
@Override
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
LOGGER.trace("start plot: {}", plotSettings);
final String tmpSubDir = ConcretePlotter.uniqueDirectoryName();
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
try {
Files.createDirectories(tmpDir);
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
final String query = plotSettings.getQuery();
final List<String> groupBy = plotSettings.getGroupBy();
final int height = plotSettings.getHeight();
final int width = plotSettings.getWidth();
final OffsetDateTime dateFrom = plotSettings.dateFrom();
final OffsetDateTime dateTo = plotSettings.dateTo();
final Result result = db.get(query, groupBy);
final long start = System.nanoTime();
final AtomicInteger idCounter = new AtomicInteger(0);
result.getGroups().stream().parallel().forEach(groupResult -> {
try {
final int id = idCounter.getAndIncrement();
final FileBackedDataSeries dataSerie = toCsv(id, groupResult, tmpDir, dateFrom, dateTo,
plotSettings);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
} catch (final Exception e) {
throw new IllegalStateException(e); // TODO
// handle
}
});
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
if (dataSeries.isEmpty()) {
throw new NoDataPointsException();
}
final Limit limitBy = plotSettings.getLimitBy();
final int limit = plotSettings.getLimit();
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
DataSeries.setColors(dataSeries);
final Path outputFile = Files.createTempFile(outputDir, "out", ".png");
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
gnuplotSettings.setHeight(height);
gnuplotSettings.setWidth(width);
defineXAxis(gnuplotSettings);
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
gnuplotSettings.setAggregate(plotSettings.getAggregate());
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
gnuplot.plot(gnuplotSettings, dataSeries);
return new PlotResult(outputFile, dataSeries, null); // TODO thumbail
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Plotting was interrupted.");
} catch (final IOException e) {
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
} finally {
FileUtils.delete(tmpDir);
LOGGER.trace("done plot");
}
}
private FileBackedDataSeries toCsv(final int id, final GroupResult groupResult, final Path tmpDir,
final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings)
throws IOException {
final long start = System.nanoTime();
final Stream<Entry> entries = groupResult.asStream();
int count = 0;
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
long ignoredValues = 0;
final char separator = ',';
final char newline = '\n';
final IntList values = new IntList(); // TODO should be a LongList
long maxValue = 0;
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
final Entry entry = it.next();
final long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
final long value = entry.getValue();
values.add((int) value);
count++;
} else {
ignoredValues++;
}
}
values.parallelSort();
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (values.size() > 0) {
// compute the percentiles
for (int i = 0; i < 100; i++) {
data.append(i);
data.append(separator);
data.append(values.get((int) Math.floor(values.size() / 100.0 * i)));
data.append(newline);
}
maxValue = values.get(values.size() - 1);
data.append(100);
data.append(separator);
data.append(maxValue);
data.append(newline);
}
output.write(data.toString());
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) grouping={}", count,
(System.nanoTime() - start) / 1_000_000.0, ignoredValues, groupResult.getGroupedBy());
final String title = ConcretePlotter.title(groupResult.getGroupedBy(), values.size());
final CsvSummary csvSummary = new CsvSummary(dataFile, values.size(), maxValue, null);
return new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.LINE);
}
private void defineXAxis(final GnuplotSettings gnuplotSettings) {
final XAxisSettings xAxis = gnuplotSettings.getxAxisSettings();
xAxis.setxDataTime(false);
xAxis.setFrom("0");
xAxis.setTo("100");
xAxis.setRotateXAxisLabel(0);
xAxis.setFormatX("%.0f");
xAxis.setXlabel("Percentile");
}
}

23
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -5,7 +5,6 @@ import java.nio.file.LinkOption;
import java.nio.file.Path;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.PlotType;
import org.lucares.performance.db.PerformanceDb;
public class Plotter {
@@ -32,26 +31,10 @@ public class Plotter {
}
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
PlotType plotType = plotSettings.getPlotType();
final ConcretePlotter plotter;
switch (plotType) {
case SCATTER:
plotter = new ScatterPlot(db, tmpBaseDir, outputDir);
break;
case PERCENTILES:
plotter = new PercentilePlot(db, tmpBaseDir, outputDir);
break;
default:
throw new UnsupportedOperationException("plot of type " + plotType + " not supported.");
}
final ScatterPlot plotter = new ScatterPlot(db, tmpBaseDir, outputDir);
return plotter.plot(plotSettings);
}
}

39
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java
View File

@@ -11,12 +11,14 @@ import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
@@ -24,6 +26,7 @@ import java.util.stream.Stream;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
@@ -32,11 +35,13 @@ import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ScatterPlot implements ConcretePlotter {
public class ScatterPlot {
private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter");
static final String DEFAULT_GROUP = "<none>";
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
@@ -58,12 +63,11 @@ public class ScatterPlot implements ConcretePlotter {
return outputDir;
}
@Override
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
LOGGER.trace("start plot: {}", plotSettings);
final String tmpSubDir = ConcretePlotter.uniqueDirectoryName();
final String tmpSubDir = uniqueDirectoryName();
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
try {
Files.createDirectories(tmpDir);
@@ -85,7 +89,7 @@ public class ScatterPlot implements ConcretePlotter {
final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
final int id = idCounter.incrementAndGet();
final String title = ConcretePlotter.title(groupResult.getGroupedBy(), csvSummary.getValues());
final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
final DataSeries dataSerie = new FileBackedDataSeries(id, title, csvSummary,
GnuplotLineType.Points);
if (dataSerie.getValues() > 0) {
@@ -237,4 +241,31 @@ public class ScatterPlot implements ConcretePlotter {
return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData());
}
static String uniqueDirectoryName() {
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
+ UUID.randomUUID().toString();
}
static String title(final Tags tags, final int values) {
final StringBuilder result = new StringBuilder();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append(" (");
result.append(String.format("%,d", values));
result.append(")");
return result.toString();
}
}