andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

extract computation of percentiles

Browse Source 
so that I can reuse the code for box plots
This commit is contained in:
Andreas Huber
2022-10-09 17:01:53 +02:00
parent ec4631e65c
commit bea6096441
2 changed files with 93 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java
View File

@@ -8,90 +8,27 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Locale;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
import org.lucares.pdb.api.RuntimeIOException;
public class CumulativeDistributionCustomAggregator implements CustomAggregator {
private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0;
private long maxValue = 0;
private final Percentiles percentiles = new Percentiles(POINTS);
private final double stepSize;
private double lastPercentile;
private double nextPercentile;
private final long totalValues;
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
final String percentile = String.format(Locale.US, "%.3f", currentPercentile);
percentiles.put(percentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
public Percentiles getPercentiles() {
return percentiles;
}
public void collect(final LongLongHashMap map) {
map.forEachOrdered(this);
percentiles.put("100.000", maxValue);
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0;
private final Path tmpDir;
private final PercentilesAggregator percentilesAggregator;
public CumulativeDistributionCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir;
percentilesAggregator = new PercentilesAggregator();
}
@Override
public void addValue(final long epochMilli, final long value) {
map.compute(value, 0, (__, l) -> l + 1);
totalValues++;
percentilesAggregator.addValue(epochMilli, value);
}
public Percentiles getPercentiles() {
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
toPercentiles.collect(map);
final Percentiles result = toPercentiles.getPercentiles();
return result;
return percentilesAggregator.getPercentiles();
}
@Override
@@ -100,17 +37,14 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
final char separator = ',';
final char newline = '\n';
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
toPercentiles.collect(map);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (map.size() > 0) {
if (percentilesAggregator.hasValues()) {
// compute the percentiles
toPercentiles.getPercentiles().forEach((percentile, value) -> {
percentilesAggregator.getPercentiles().forEach((percentile, value) -> {
data.append(percentile);
data.append(separator);

86
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentilesAggregator.java Normal file
View File

@@ -0,0 +1,86 @@
package org.lucares.pdb.plot.api;
import java.util.Locale;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
public class PercentilesAggregator {
private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0;
private long maxValue = 0;
private final Percentiles percentiles = new Percentiles(POINTS);
private final double stepSize;
private double lastPercentile;
private double nextPercentile;
private final long totalValues;
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
final String percentile = String.format(Locale.US, "%.3f", currentPercentile);
percentiles.put(percentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
public Percentiles getPercentiles() {
return percentiles;
}
public void collect(final LongLongHashMap map) {
map.forEachOrdered(this);
percentiles.put("100.000", maxValue);
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0;
public PercentilesAggregator() {
}
public void addValue(final long epochMilli, final long value) {
map.compute(value, 0, (__, l) -> l + 1);
totalValues++;
}
public Percentiles getPercentiles() {
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
toPercentiles.collect(map);
final Percentiles result = toPercentiles.getPercentiles();
return result;
}
public boolean hasValues() {
return map.size() > 0;
}
}