diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java index 9f49073..04c22ed 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentileCustomAggregator.java @@ -11,22 +11,20 @@ import java.nio.file.Path; import org.lucares.collections.IntList; -public class PercentileCustomAggregator implements CustomAggregator{ +public class PercentileCustomAggregator implements CustomAggregator { + + private final static int POINTS = 500; - private final static int POINTS = 100; - private final IntList values = new IntList(); // TODO should be a LongList - private Path tmpDir; + private final Path tmpDir; - - - public PercentileCustomAggregator(Path tmpDir) { + public PercentileCustomAggregator(final Path tmpDir) { this.tmpDir = tmpDir; } @Override - public void addValue(long epochMilli, long value) { + public void addValue(final long epochMilli, final long value) { values.add((int) value); } @@ -34,24 +32,24 @@ public class PercentileCustomAggregator implements CustomAggregator{ public AggregatedData getAggregatedData() throws IOException { final char separator = ','; final char newline = '\n'; - + values.parallelSort(); - + final IntList percentiles = new IntList(POINTS); final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){ - + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final StringBuilder data = new StringBuilder(); if (values.size() > 0) { // compute the percentiles for (int i = 0; i < POINTS; i++) { - data.append(i* (100/(double)POINTS)); + data.append(i * (100 / (double) POINTS)); data.append(separator); - int percentile = values.get((int) Math.floor(values.size() - / ((double)POINTS) * i)); + final int percentile = values.get((int) Math.floor(values.size() / ((double) POINTS) * i)); data.append(percentile); data.append(newline); - + percentiles.add(percentile); } final int maxValue = values.get(values.size() - 1); @@ -61,12 +59,11 @@ public class PercentileCustomAggregator implements CustomAggregator{ data.append(newline); } output.write(data.toString()); - + } // TODO remove: - double average = percentiles.stream().summaryStatistics().getAverage(); - - + final double average = percentiles.stream().summaryStatistics().getAverage(); + final String title = String.format("percentiles"); return new AggregatedData(title, dataFile, average); } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java index aa17ba7..9ee0a25 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java @@ -205,9 +205,15 @@ public class ScatterPlot { final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); + + final long minValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? 0 + : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMin()); + final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE + : plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax()); + final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir); - long maxValue = 0; + long statsMaxValue = 0; long ignoredValues = 0; final int separator = ','; final int newline = '\n'; @@ -221,39 +227,44 @@ public class ScatterPlot { final Entry entry = it.next(); final long epochMilli = entry.getEpochMilli(); - if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) { - - final long value = entry.getValue(); - final String stringValue = LongUtils.longToString(value); - final String formattedDate; - - if (useMillis) { - formattedDateBuilder.delete(0, formattedDateBuilder.length()); - formatter.format("%.3f", epochMilli / 1000.0); - formattedDate = formattedDateBuilder.toString(); - } else { - formattedDate = String.valueOf(epochMilli / 1000); - } - - output.write(formattedDate); - output.write(separator); - output.write(stringValue); - output.write(newline); - - aggregator.addValue(epochMilli, value); - - count++; - maxValue = Math.max(maxValue, value); - } else { + if (fromEpochMilli > epochMilli || epochMilli > toEpochMilli) { ignoredValues++; + continue; } + + final long value = entry.getValue(); + aggregator.addValue(epochMilli, value); + if (value < minValue || value > maxValue) { + ignoredValues++; + continue; + } + + final String stringValue = LongUtils.longToString(value); + final String formattedDate; + + if (useMillis) { + formattedDateBuilder.delete(0, formattedDateBuilder.length()); + formatter.format("%.3f", epochMilli / 1000.0); + formattedDate = formattedDateBuilder.toString(); + } else { + formattedDate = String.valueOf(epochMilli / 1000); + } + + output.write(formattedDate); + output.write(separator); + output.write(stringValue); + output.write(newline); + + count++; + statsMaxValue = Math.max(statsMaxValue, value); + } } METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy().asString(), dataFile); - return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData()); + return new CsvSummary(dataFile, count, statsMaxValue, aggregator.getAggregatedData()); } static String uniqueDirectoryName() {