andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add plots for percentiles

Browse Source
This commit is contained in:
ahr
2017-11-06 16:57:22 +01:00
parent 92dde94443
commit 64db4c48a2
19 changed files with 301 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pdb-plotting/build.gradle
View File

@@ -1,6 +1,7 @@
dependencies {
compile project(':performanceDb')
compile 'org.lucares:primitiveCollections:0.1.20171007100354'
compile 'com.fasterxml.jackson.core:jackson-databind:2.9.1'
compile 'com.google.guava:guava:23.0'
}

5
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java
View File

@@ -6,11 +6,14 @@ import org.lucares.recommind.logs.DataSeries;
public interface AggregateHandler {
void addStats(StringBuilder result, Collection<DataSeries> dataSeries);
void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries);
default void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(format + "\n", args));
}
void addPlots(StringBuilder result, Collection<DataSeries> dataSeries);
CustomAggregator createCustomAggregator(long minDate, long maxDate);
}

21
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java Normal file
View File

@@ -0,0 +1,21 @@
package org.lucares.pdb.plot.api;
import java.util.List;
public class AggregatedData {
private final String label;
private final List<AggregatedDataEntry> data;
public AggregatedData(String label, List<AggregatedDataEntry> data) {
super();
this.label = label;
this.data = data;
}
public String getLabel() {
return label;
}
public List<AggregatedDataEntry> getData() {
return data;
}
}

18
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataEntry.java Normal file
View File

@@ -0,0 +1,18 @@
package org.lucares.pdb.plot.api;
public class AggregatedDataEntry {
private final double epochSeconds;
private final long value;
public AggregatedDataEntry(double epochSeconds, long value) {
super();
this.epochSeconds = epochSeconds;
this.value = value;
}
public double getEpochSeconds() {
return epochSeconds;
}
public long getValue() {
return value;
}
}

9
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java Normal file
View File

@@ -0,0 +1,9 @@
package org.lucares.pdb.plot.api;
public interface CustomAggregator {
void addValue(long epochMilli, long value);
AggregatedData getAggregatedData();
}

8
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/MeanAggregate.java
View File

@@ -9,7 +9,7 @@ public class MeanAggregate implements AggregateHandler{
@Override
public void addStats(StringBuilder result, Collection<DataSeries> dataSeries) {
public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
int count = 1;
for (final DataSeries dataSerie : dataSeries) {
@@ -24,7 +24,13 @@ public class MeanAggregate implements AggregateHandler{
for (final DataSeries dataSerie : dataSeries) {
appendfln(result, "A%d_mean title '%s Mean', \\", count,
dataSerie.getTitle(), dataSerie.getTitle());
count++;
}
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
return new NullCustomAggregator();
}
}

7
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullAggregate.java
View File

@@ -8,7 +8,7 @@ public class NullAggregate implements AggregateHandler {
@Override
public void addStats(StringBuilder result, Collection<DataSeries> dataSeries) {
public void addGnuplotDefinitions(StringBuilder result, String separator ,Collection<DataSeries> dataSeries) {
// nothing to do; this is a Null-Object
}
@@ -17,4 +17,9 @@ public class NullAggregate implements AggregateHandler {
// nothing to do; this is a Null-Object
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
return new NullCustomAggregator();
}
}

16
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/NullCustomAggregator.java Normal file
View File

@@ -0,0 +1,16 @@
package org.lucares.pdb.plot.api;
public class NullCustomAggregator implements CustomAggregator{
@Override
public void addValue(long epochMilli, long value) {
// TODO Auto-generated method stub
}
@Override
public AggregatedData getAggregatedData() {
return null;
}
}

50
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileAggregate.java Normal file
View File

@@ -0,0 +1,50 @@
package org.lucares.pdb.plot.api;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
public class PercentileAggregate implements AggregateHandler{
private double percentile;
/**
*
* @param percentile range &gt; 0.0 and &lt; 1.0)
*/
public PercentileAggregate(double percentile) {
this.percentile = percentile;
}
@Override
public void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries) {
for (DataSeries dataSerie : dataSeries) {
if (dataSerie.getAggregatedData() != null){
appendfln(result, "$%s << EOD", dataSerie.getId());
for (AggregatedDataEntry aggregatedData : dataSerie.getAggregatedData().getData()) {
appendfln(result, "%.3f%s%d", aggregatedData.getEpochSeconds(), separator, aggregatedData.getValue());
}
appendfln(result, "EOD", dataSerie.getId());
}
}
}
@Override
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
for (DataSeries dataSerie : dataSeries) {
appendfln(result, "'$%s' using 1:2 title '%s' with line, \\", dataSerie.getId(), dataSerie.getTitle()+" " +dataSerie.getAggregatedData().getLabel());
}
}
@Override
public CustomAggregator createCustomAggregator(long minDate, long maxDate) {
return new PercentileCustomAggregator(percentile, minDate, maxDate);
}
}

54
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java Normal file
View File

@@ -0,0 +1,54 @@
package org.lucares.pdb.plot.api;
import java.util.ArrayList;
import java.util.List;
import org.lucares.collections.IntList;
public class PercentileCustomAggregator implements CustomAggregator{
private double percentile;
private long fromEpochMilli;
private long numBuckets;
private long bucketSize;
private List<IntList> buckets = new ArrayList<>();
public PercentileCustomAggregator(double percentile, long fromEpochMilli, long toEpochMilli) {
this.percentile = percentile;
this.fromEpochMilli = fromEpochMilli;
this.numBuckets = 50;
this.bucketSize = (toEpochMilli - fromEpochMilli) / numBuckets;
for(int i = 0; i < numBuckets; i++){
buckets.add(new IntList());
}
}
@Override
public void addValue(long epochMilli, long value) {
final int bucket = (int) ((epochMilli - fromEpochMilli ) / bucketSize);
buckets.get(bucket).add((int)value); // TODO we should use a LongList instead of an IntList
}
@Override
public AggregatedData getAggregatedData() {
final List<AggregatedDataEntry> data = new ArrayList<>((int)numBuckets);
for (int i= 0; i < numBuckets; i++) {
final IntList values = buckets.get(i);
values.sort();
final int indexOfPercentile = (int) (values.size() * percentile);
final long value = values.size() > indexOfPercentile ? values.get(indexOfPercentile) : 0;
final long epochMilli = fromEpochMilli + i*bucketSize + bucketSize/2; // the middle of the bucket
final double epochSecond = epochMilli / 1000.0; // gnuplot wants the time in seconds
final AggregatedDataEntry aggregatedDataEntry = new AggregatedDataEntry(epochSecond, value);
if (value > 0){
data.add(aggregatedDataEntry );
}
}
final String title = String.format("%.1f%% percentile", percentile*100);
return new AggregatedData(title, data);
}
}

10
pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java
View File

@@ -2,17 +2,21 @@ package org.lucares.recommind.logs;
import java.io.File;
import org.lucares.pdb.plot.api.AggregatedData;
class CsvSummary {
private final int values;
private long maxValue;
private File dataFile;
private AggregatedData aggregatedData;
public CsvSummary(File dataFile, final int values, long maxValue) {
public CsvSummary(File dataFile, final int values, long maxValue, AggregatedData aggregatedData) {
super();
this.dataFile = dataFile;
this.values = values;
this.maxValue = maxValue;
this.aggregatedData = aggregatedData;
}
public File getDataFile() {
@@ -26,4 +30,8 @@ class CsvSummary {
public long getMaxValue() {
return maxValue;
}
public AggregatedData getAggregatedData() {
return aggregatedData;
}
}

146
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -1,76 +1,70 @@
package org.lucares.recommind.logs;
import java.io.File;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class DataSeries {
public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (a, b) -> {
return a.getValues() - b.getValues();
};
public static final Comparator<? super DataSeries> BY_MAX_VALUE = (a, b) -> {
final long result = a.getMaxValue() - b.getMaxValue();
return result <0 ? -1 : (result > 0 ? 1 : 0);
};
private final File dataFile;
private final String title;
private final GnuplotColor color;
private final Integer pointType;
private final int values;
private long maxValue;
public DataSeries(final File dataFile, final String title, final int values, long maxValue) {
super();
this.dataFile = dataFile;
this.title = title;
this.values = values;
this.maxValue = maxValue;
this.color = null;
this.pointType = null;
}
public GnuplotColor getColor() {
return color;
}
public Integer getPointType() {
return pointType;
}
public File getDataFile() {
return dataFile;
}
public String getTitle() {
return title;
}
public int getValues() {
return values;
}
public long getMaxValue() {
return maxValue;
}
public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
final Map<String, Integer> result = new LinkedHashMap<>();
for (final DataSeries dataSerie : dataSeries) {
result.put(dataSerie.getTitle(), dataSerie.values);
}
return result;
}
}
package org.lucares.recommind.logs;
import java.io.File;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.plot.api.AggregatedData;
public class DataSeries {
public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (
a, b) -> {
return a.getValues() - b.getValues();
};
public static final Comparator<? super DataSeries> BY_MAX_VALUE = (a, b) -> {
final long result = a.getMaxValue() - b.getMaxValue();
return result < 0 ? -1 : (result > 0 ? 1 : 0);
};
private final String title;
private CsvSummary csvSummary;
private String id;
public DataSeries(String id, String title, CsvSummary csvSummary) {
this.id = id;
this.title = title;
this.csvSummary = csvSummary;
}
public String getId() {
return id;
}
public File getDataFile() {
return csvSummary.getDataFile();
}
public String getTitle() {
return title;
}
public int getValues() {
return csvSummary.getValues();
}
public long getMaxValue() {
return csvSummary.getMaxValue();
}
public AggregatedData getAggregatedData() {
return csvSummary.getAggregatedData();
}
public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
final Map<String, Integer> result = new LinkedHashMap<>();
for (final DataSeries dataSerie : dataSeries) {
result.put(dataSerie.getTitle(), dataSerie.getValues());
}
return result;
}
}

3
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -2,7 +2,6 @@ package org.lucares.recommind.logs;
import java.util.Collection;
import org.lucares.pdb.plot.api.AggreateInternal;
import org.lucares.pdb.plot.api.AxisScale;
public class GnuplotFileGenerator {
@@ -18,7 +17,7 @@ public class GnuplotFileGenerator {
appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator());
settings.getAggregate().addStats(result, dataSeries);
settings.getAggregate().addGnuplotDefinitions(result, settings.getDatafileSeparator(), dataSeries);
appendfln(result, "set timefmt '%s'", settings.getTimefmt());

1
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java
View File

@@ -2,7 +2,6 @@ package org.lucares.recommind.logs;
import java.nio.file.Path;
import org.lucares.pdb.plot.api.AggreateInternal;
import org.lucares.pdb.plot.api.AggregateHandler;
import org.lucares.pdb.plot.api.AxisScale;

28
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -26,8 +26,7 @@ import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.AggreateInternal;
import org.lucares.pdb.plot.api.AggregateHandler;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.performance.db.PerformanceDb;
@@ -92,6 +91,7 @@ public class Plotter {
final Result result = db.get(query, groupBy);
int idCounter = 0;
for (final GroupResult groupResult : result.getGroups()) {
final Stream<Entry> entries = groupResult.asStream();
@@ -99,10 +99,11 @@ public class Plotter {
final CsvSummary csvSummary = toCsv(entries, tmpDir, dateFrom, dateTo, plotSettings);
final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
final DataSeries dataSerie = new DataSeries(csvSummary.getDataFile(), title, csvSummary.getValues(), csvSummary.getMaxValue());
final DataSeries dataSerie = new DataSeries("id"+idCounter, title, csvSummary);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
idCounter++;
}
if (dataSeries.isEmpty()) {
@@ -233,8 +234,7 @@ public class Plotter {
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final AggregateHandler aggregate = plotSettings.getAggregate();
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
long maxValue = 0;
long ignoredValues = 0;
@@ -248,26 +248,30 @@ public class Plotter {
while (it.hasNext()) {
final Entry entry = it.next();
if (fromEpochMilli <= entry.getEpochMilli() && entry.getEpochMilli() <= toEpochMilli) {
long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
final String value = longToString(entry.getValue());
long value = entry.getValue();
final String stringValue = longToString(value);
final String formattedDate;
if (useMillis){
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", entry.getEpochMilli() / 1000.0);
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
}else {
formattedDate = String.valueOf(entry.getEpochMilli() / 1000);
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(value);
output.write(stringValue);
output.write(newline);
aggregator.addValue(epochMilli, value);
count++;
maxValue = Math.max(maxValue, entry.getValue());
maxValue = Math.max(maxValue, value);
}else {
ignoredValues++;
}
@@ -275,7 +279,7 @@ public class Plotter {
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis));
return new CsvSummary(dataFile, count, maxValue);
return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData());
}
private static String longToString(final long value){