extract computation of percentiles

so that I can reuse the code for box plots
This commit is contained in:
2022-10-09 17:01:53 +02:00
parent ec4631e65c
commit bea6096441
2 changed files with 93 additions and 73 deletions

View File

@@ -8,90 +8,27 @@ import java.io.OutputStreamWriter;
import java.io.Writer; import java.io.Writer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Locale;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
import org.lucares.pdb.api.RuntimeIOException; import org.lucares.pdb.api.RuntimeIOException;
public class CumulativeDistributionCustomAggregator implements CustomAggregator { public class CumulativeDistributionCustomAggregator implements CustomAggregator {
private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0;
private long maxValue = 0;
private final Percentiles percentiles = new Percentiles(POINTS);
private final double stepSize;
private double lastPercentile;
private double nextPercentile;
private final long totalValues;
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
final String percentile = String.format(Locale.US, "%.3f", currentPercentile);
percentiles.put(percentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
public Percentiles getPercentiles() {
return percentiles;
}
public void collect(final LongLongHashMap map) {
map.forEachOrdered(this);
percentiles.put("100.000", maxValue);
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0;
private final Path tmpDir; private final Path tmpDir;
private final PercentilesAggregator percentilesAggregator;
public CumulativeDistributionCustomAggregator(final Path tmpDir) { public CumulativeDistributionCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir; this.tmpDir = tmpDir;
percentilesAggregator = new PercentilesAggregator();
} }
@Override @Override
public void addValue(final long epochMilli, final long value) { public void addValue(final long epochMilli, final long value) {
map.compute(value, 0, (__, l) -> l + 1); percentilesAggregator.addValue(epochMilli, value);
totalValues++;
} }
public Percentiles getPercentiles() { public Percentiles getPercentiles() {
final ToPercentiles toPercentiles = new ToPercentiles(totalValues); return percentilesAggregator.getPercentiles();
toPercentiles.collect(map);
final Percentiles result = toPercentiles.getPercentiles();
return result;
} }
@Override @Override
@@ -100,17 +37,14 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
final char separator = ','; final char separator = ',';
final char newline = '\n'; final char newline = '\n';
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
toPercentiles.collect(map);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter( try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder(); final StringBuilder data = new StringBuilder();
if (map.size() > 0) { if (percentilesAggregator.hasValues()) {
// compute the percentiles // compute the percentiles
toPercentiles.getPercentiles().forEach((percentile, value) -> { percentilesAggregator.getPercentiles().forEach((percentile, value) -> {
data.append(percentile); data.append(percentile);
data.append(separator); data.append(separator);

View File

@@ -0,0 +1,86 @@
package org.lucares.pdb.plot.api;
import java.util.Locale;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
public class PercentilesAggregator {
private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0;
private long maxValue = 0;
private final Percentiles percentiles = new Percentiles(POINTS);
private final double stepSize;
private double lastPercentile;
private double nextPercentile;
private final long totalValues;
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
final String percentile = String.format(Locale.US, "%.3f", currentPercentile);
percentiles.put(percentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
public Percentiles getPercentiles() {
return percentiles;
}
public void collect(final LongLongHashMap map) {
map.forEachOrdered(this);
percentiles.put("100.000", maxValue);
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0;
public PercentilesAggregator() {
}
public void addValue(final long epochMilli, final long value) {
map.compute(value, 0, (__, l) -> l + 1);
totalValues++;
}
public Percentiles getPercentiles() {
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
toPercentiles.collect(map);
final Percentiles result = toPercentiles.getPercentiles();
return result;
}
public boolean hasValues() {
return map.size() > 0;
}
}