andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

replace individual percentile aggregates with a single one for all

Browse Source
This commit is contained in:
Andreas Huber
2018-01-21 10:54:13 +01:00
parent b439c9d79a
commit 8f15aba0d5
12 changed files with 99 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java
View File

@@ -1,5 +1,6 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
@@ -16,6 +17,6 @@ public interface AggregateHandler {
void addPlots(StringBuilder result, Collection<DataSeries> dataSeries);
CustomAggregator createCustomAggregator(long minDate, long maxDate);
CustomAggregator createCustomAggregator(Path tmpDir);
}

28
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java
View File

@@ -1,21 +1,21 @@
package org.lucares.pdb.plot.api;
import java.util.List;
import java.io.File;
public class AggregatedData {
private final String label;
private final List<AggregatedDataEntry> data;
public AggregatedData(String label, List<AggregatedDataEntry> data) {
super();
this.label = label;
this.data = data;
}
public String getLabel() {
return label;
}
public List<AggregatedDataEntry> getData() {
return data;
}
private final String label;
private File dataFile;
public AggregatedData(String label, File dataFile) {
this.label = label;
this.dataFile = dataFile;
}
public String getLabel() {
return label;
}
public File getDataFile() {
return dataFile;
}
}

4
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java
View File

@@ -1,9 +1,11 @@
package org.lucares.pdb.plot.api;
import java.io.IOException;
public interface CustomAggregator {
void addValue(long epochMilli, long value);
AggregatedData getAggregatedData();
AggregatedData getAggregatedData() throws IOException;
}

38
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java
View File

@@ -1,38 +0,0 @@
package org.lucares.pdb.plot.api;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.FileBackedDataSeries;
public class MeanAggregate implements AggregateHandler{
@Override
public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
int count = 1;
for (final DataSeries dataSerie : dataSeries) {
if (dataSerie instanceof FileBackedDataSeries){
appendfln(result, "stats '%s' using 2 prefix \"A%d\"", ((FileBackedDataSeries) dataSerie).getDataFile(),count);
count++;
}
}
}
@Override
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
int count = 1;
for (final DataSeries dataSerie : dataSeries) {
if (dataSerie instanceof FileBackedDataSeries){
appendfln(result, "A%d_mean title '%s Mean', \\", count,
dataSerie.getTitle(), dataSerie.getTitle());
count++;
}
}
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
return new NullCustomAggregator();
}
}

7
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java
View File

@@ -1,14 +1,15 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
public class NullAggregate implements AggregateHandler {
@Override
public void addGnuplotDefinitions(StringBuilder result, String separator ,Collection<DataSeries> dataSeries) {
public void addGnuplotDefinitions(StringBuilder result, String separator,
Collection<DataSeries> dataSeries) {
// nothing to do; this is a Null-Object
}
@@ -18,7 +19,7 @@ public class NullAggregate implements AggregateHandler {
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
public CustomAggregator createCustomAggregator(Path tmpDir) {
return new NullCustomAggregator();
}

42
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java
View File

@@ -1,52 +1,38 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
public class PercentileAggregate implements AggregateHandler{
private double percentile;
/**
*
* @param percentile range &gt; 0.0 and &lt; 1.0)
*/
public PercentileAggregate(double percentile) {
this.percentile = percentile;
@Override
public CustomAggregator createCustomAggregator(Path tmpDir) {
return new PercentileCustomAggregator(tmpDir);
}
public PercentileAggregate() {
}
@Override
public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
for (DataSeries dataSerie : dataSeries) {
if (dataSerie.getAggregatedData() != null){
appendfln(result, "$%s << EOD", dataSerie.getId());
for (AggregatedDataEntry aggregatedData : dataSerie.getAggregatedData().getData()) {
appendfln(result, "%.3f%s%d", aggregatedData.getEpochSeconds(), separator, aggregatedData.getValue());
}
appendfln(result, "EOD", dataSerie.getId());
}
}
// TODO still neeeded?
}
@Override
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
for (DataSeries dataSerie : dataSeries) {
if (dataSerie.getAggregatedData() != null){
appendfln(result, "'$%s' using 1:2 title '%s' with linespoints lw 2, \\", dataSerie.getId(), dataSerie.getTitle()+" " +dataSerie.getAggregatedData().getLabel());
final AggregatedData aggregatedData = dataSerie.getAggregatedData();
if (aggregatedData != null){
appendfln(result, "'%s' using 1:2 title '%s' with lines lw 1 axes x2y1, \\", //
aggregatedData.getDataFile().getAbsolutePath(),//
dataSerie.getTitle()+" " +aggregatedData.getLabel());
}
}
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
return new PercentileCustomAggregator(percentile, minDate, maxDate);
}
}

76
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java
View File

@@ -1,54 +1,68 @@
package org.lucares.pdb.plot.api;
import java.util.ArrayList;
import java.util.List;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import org.lucares.collections.IntList;
public class PercentileCustomAggregator implements CustomAggregator{
private double percentile;
private long fromEpochMilli;
private long numBuckets;
private long bucketSize;
private final static int POINTS = 300;
private List<IntList> buckets = new ArrayList<>();
private final IntList values = new IntList(); // TODO should be a LongList
public PercentileCustomAggregator(double percentile, long fromEpochMilli, long toEpochMilli) {
this.percentile = percentile;
this.fromEpochMilli = fromEpochMilli;
this.numBuckets = 50;
this.bucketSize = (toEpochMilli - fromEpochMilli) / numBuckets;
for(int i = 0; i < numBuckets; i++){
buckets.add(new IntList());
}
private Path tmpDir;
public PercentileCustomAggregator(Path tmpDir) {
this.tmpDir = tmpDir;
}
@Override
public void addValue(long epochMilli, long value) {
final int bucket = (int) ((epochMilli - fromEpochMilli ) / bucketSize);
buckets.get(bucket).add((int)value); // TODO we should use a LongList instead of an IntList
values.add((int) value);
}
@Override
public AggregatedData getAggregatedData() {
final List<AggregatedDataEntry> data = new ArrayList<>((int)numBuckets);
public AggregatedData getAggregatedData() throws IOException {
final char separator = ',';
final char newline = '\n';
for (int i= 0; i < numBuckets; i++) {
final IntList values = buckets.get(i);
values.sort();
final int indexOfPercentile = (int) (values.size() * percentile);
final long value = values.size() > indexOfPercentile ? values.get(indexOfPercentile) : 0;
final long epochMilli = fromEpochMilli + i*bucketSize + bucketSize/2; // the middle of the bucket
final double epochSecond = epochMilli / 1000.0; // gnuplot wants the time in seconds
final AggregatedDataEntry aggregatedDataEntry = new AggregatedDataEntry(epochSecond, value);
values.parallelSort();
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
if (value > 0){
data.add(aggregatedDataEntry );
final StringBuilder data = new StringBuilder();
if (values.size() > 0) {
// compute the percentiles
for (int i = 0; i < POINTS; i++) {
data.append(i* (100/(double)POINTS));
data.append(separator);
data.append(values.get((int) Math.floor(values.size()
/ ((double)POINTS) * i)));
data.append(newline);
}
final int maxValue = values.get(values.size() - 1);
data.append(100);
data.append(separator);
data.append(maxValue);
data.append(newline);
}
output.write(data.toString());
}
final String title = String.format("%.1f%% percentile", percentile*100);
return new AggregatedData(title, data);
final String title = String.format("percentiles");
return new AggregatedData(title, dataFile);
}
}

12
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -30,6 +30,18 @@ public class GnuplotFileGenerator {
appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel());
appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo());
/*
* set yrange ["0":"100"]
set ytics nomirror # don't show the tics of the left y-axis on the right side
set y2label "CPU load"
set y2tics
set autoscale y2
*/
appendfln(result, "set x2label \"percent\"");
appendfln(result, "set y2tics");
appendfln(result, "set x2range [\"0\":\"100\"]");
final long graphOffset = settings.getYAxisScale() == AxisScale.LINEAR ? 0 : 1;
appendfln(result, "set yrange [\""+graphOffset+"\":]");

2
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java
View File

@@ -171,7 +171,7 @@ public class ScatterPlot implements ConcretePlotter {
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir);
long maxValue = 0;
long ignoredValues = 0;