andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

plot statistics written to the legend only count visible values

Browse Source
This commit is contained in:
Andreas Huber
2018-05-01 13:03:34 +02:00
parent a4d04ebece
commit 01bd7e4388
2 changed files with 54 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java
View File

@@ -11,22 +11,20 @@ import java.nio.file.Path;
import org.lucares.collections.IntList;
public class PercentileCustomAggregator implements CustomAggregator{
public class PercentileCustomAggregator implements CustomAggregator {
private final static int POINTS = 100;
private final static int POINTS = 500;
private final IntList values = new IntList(); // TODO should be a LongList
private Path tmpDir;
private final Path tmpDir;
public PercentileCustomAggregator(Path tmpDir) {
public PercentileCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir;
}
@Override
public void addValue(long epochMilli, long value) {
public void addValue(final long epochMilli, final long value) {
values.add((int) value);
}
@@ -39,16 +37,16 @@ public class PercentileCustomAggregator implements CustomAggregator{
final IntList percentiles = new IntList(POINTS);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (values.size() > 0) {
// compute the percentiles
for (int i = 0; i < POINTS; i++) {
data.append(i* (100/(double)POINTS));
data.append(i * (100 / (double) POINTS));
data.append(separator);
int percentile = values.get((int) Math.floor(values.size()
/ ((double)POINTS) * i));
final int percentile = values.get((int) Math.floor(values.size() / ((double) POINTS) * i));
data.append(percentile);
data.append(newline);
@@ -64,8 +62,7 @@ public class PercentileCustomAggregator implements CustomAggregator{
}
// TODO remove:
double average = percentiles.stream().summaryStatistics().getAverage();
final double average = percentiles.stream().summaryStatistics().getAverage();
final String title = String.format("percentiles");
return new AggregatedData(title, dataFile, average);

63
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java
View File

@@ -205,9 +205,15 @@ public class ScatterPlot {
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin());
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir);
long maxValue = 0;
long statsMaxValue = 0;
long ignoredValues = 0;
final int separator = ',';
final int newline = '\n';
@@ -221,39 +227,44 @@ public class ScatterPlot {
final Entry entry = it.next();
final long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
final long value = entry.getValue();
final String stringValue = LongUtils.longToString(value);
final String formattedDate;
if (useMillis) {
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
} else {
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(stringValue);
output.write(newline);
aggregator.addValue(epochMilli, value);
count++;
maxValue = Math.max(maxValue, value);
} else {
if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) {
ignoredValues++;
continue;
}
final long value = entry.getValue();
aggregator.addValue(epochMilli, value);
if (value < minValue || value > maxValue) {
ignoredValues++;
continue;
}
final String stringValue = LongUtils.longToString(value);
final String formattedDate;
if (useMillis) {
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
} else {
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(stringValue);
output.write(newline);
count++;
statsMaxValue = Math.max(statsMaxValue, value);
}
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}",
count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
groupResult.getGroupedBy().asString(), dataFile);
return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData());
return new CsvSummary(dataFile, count, statsMaxValue, aggregator.getAggregatedData());
}
static String uniqueDirectoryName() {