make it possible to render any combination of plots

This commit is contained in:
2019-10-20 10:16:25 +02:00
parent b7c4fe4c1f
commit f28a67a5c1
26 changed files with 321 additions and 243 deletions

View File

@@ -1,5 +0,0 @@
package org.lucares.pdb.plot.api;
public enum AggreateInternal {
NONE, MEAN, PERCENTILE95, PERCENTILE99, PERCENTILE999
}

View File

@@ -0,0 +1,17 @@
package org.lucares.pdb.plot.api;
/**
* Note: The order in this enum defines the order in which the aggregates are drawn.
*/
public enum Aggregate {
PARALLEL,
SCATTER,
/**
* Empirical cumulative distribution functions
*
* @see https://serialmentor.com/dataviz/ecdf-qq.html
*/
CUM_DISTRIBUTION,
}

View File

@@ -2,12 +2,14 @@ package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.LineStyle;
public interface AggregateHandler {
// TODO handle aggregates as normal DataSeries
Aggregate getAggregateType();
void addGnuplotDefinitions(StringBuilder result, String separator, Collection<DataSeries> dataSeries);
@@ -18,10 +20,17 @@ public interface AggregateHandler {
default void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(format + "\n", args));
}
default String gnuplotTitle(Optional<String> title) {
return title.isPresent() ? "title '"+title.get()+"'": "notitle";
}
void addPlotsBeforeScatter(StringBuilder result, Collection<DataSeries> dataSeries);
void addPlotsAfterScatter(StringBuilder result, Collection<DataSeries> dataSeries);
// void addPlotsBeforeScatter(StringBuilder result, Collection<DataSeries> dataSeries);
//
// void addPlotsAfterScatter(StringBuilder result, Collection<DataSeries> dataSeries);
void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Optional<String> title);
CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, long toEpochMilli);

View File

@@ -0,0 +1,64 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import org.lucares.recommind.logs.DataSeries;
public class AggregateHandlerCollection {
private static final Comparator<AggregateHandler> COMPARATOR = Comparator.comparing(AggregateHandler::getAggregateType);
private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
public void add(AggregateHandler aggregateHandler) {
aggregateHandlers.add(aggregateHandler);
Collections.sort(aggregateHandlers, COMPARATOR);
}
public void addGnuplotDefinitions(StringBuilder result, String datafileSeparator, Collection<DataSeries> dataSeries) {
for (AggregateHandler handler : aggregateHandlers) {
handler.addGnuplotDefinitions(result, datafileSeparator, dataSeries);
}
}
public AggregatorCollection createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli) {
final List<CustomAggregator> aggregators = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
CustomAggregator aggregator = handler.createCustomAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli);
if (aggregator != null) {
aggregators.add(aggregator);
}
}
return new AggregatorCollection(aggregators);
}
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
boolean first = true;
for (AggregateHandler handler : aggregateHandlers) {
for (DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty();
Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData().get(handler.getAggregateType());
if(aggregatedData.isPresent()) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title);
}
}
first = false;
}
}
}

View File

@@ -0,0 +1,22 @@
package org.lucares.pdb.plot.api;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Optional;
public class AggregatedDataCollection implements Iterable<AggregatedData>{
private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>();
public void put(Aggregate aggregate, AggregatedData aggregatedData) {
aggregatedDatas.put(aggregate, aggregatedData);
}
@Override
public Iterator<AggregatedData> iterator() {
return aggregatedDatas.values().iterator();
}
public Optional<AggregatedData> get(Aggregate aggregateType) {
return Optional.ofNullable(aggregatedDatas.get(aggregateType));
}
}

View File

@@ -0,0 +1,29 @@
package org.lucares.pdb.plot.api;
import java.io.IOException;
import java.util.List;
public class AggregatorCollection {
private final List<CustomAggregator> aggregators;
public AggregatorCollection(List<CustomAggregator> aggregators) {
this.aggregators = aggregators;
}
public void addValue(boolean valueIsInYRange, long epochMilli, long value) {
for (CustomAggregator aggregator : aggregators) {
aggregator.addValue(valueIsInYRange, epochMilli, value);
}
}
public AggregatedDataCollection getAggregatedData() throws IOException {
AggregatedDataCollection result = new AggregatedDataCollection();
for (CustomAggregator aggregator : aggregators) {
result.put(aggregator.getType(), aggregator.getAggregatedData());
}
return result;
}
}

View File

@@ -120,4 +120,9 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
return new AggregatedData(title, dataFile);
}
@Override
public Aggregate getType() {
return Aggregate.CUM_DISTRIBUTION;
}
}

View File

@@ -2,49 +2,44 @@ package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.LineStyle;
public class CumulativeDistributionHandler implements AggregateHandler {
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, final long fromEpochMilli,
final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir);
}
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir);
}
public CumulativeDistributionHandler() {
}
public CumulativeDistributionHandler() {
}
@Override
public void addGnuplotDefinitions(final StringBuilder result, final String separator,
final Collection<DataSeries> dataSeries) {
@Override
public void addGnuplotDefinitions(final StringBuilder result, final String separator,
final Collection<DataSeries> dataSeries) {
appendln(result, "set x2label \"Cumulative Distribution\"");
appendln(result, "set format x2 \"%.0f%%\"");
appendln(result, "set x2tics 5");
appendln(result, "set x2range [\"0\":\"100\"]");
}
appendln(result, "set x2label \"Cumulative Distribution\"");
appendln(result, "set format x2 \"%.0f%%\"");
appendln(result, "set x2tics 5");
appendln(result, "set x2range [\"0\":\"100\"]");
}
@Override
public void addPlotsBeforeScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
// nothing to do: the percentile line should be drawn on the scatter plots and
// must therefore be rendered after the scatter
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with lines axes x2y1 lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title),//
lineStyle.darker()//
);
}
@Override
public void addPlotsAfterScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
for (final DataSeries dataSerie : dataSeries) {
final AggregatedData aggregatedData = dataSerie.getAggregatedData();
if (aggregatedData != null) {
final LineStyle lineStyle = dataSerie.getStyle();
appendfln(result, "'%s' using 1:2 notitle with lines axes x2y1 lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
lineStyle.darker()//
);
}
}
}
@Override
public Aggregate getAggregateType() {
return Aggregate.CUM_DISTRIBUTION;
}
}

View File

@@ -7,4 +7,6 @@ public interface CustomAggregator {
void addValue(boolean valueIsInYRange, long epochMilli, long value);
AggregatedData getAggregatedData() throws IOException;
Aggregate getType();
}

View File

@@ -1,32 +0,0 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import org.lucares.recommind.logs.DataSeries;
public class NullAggregate implements AggregateHandler {
@Override
public void addGnuplotDefinitions(final StringBuilder result, final String separator,
final Collection<DataSeries> dataSeries) {
// nothing to do; this is a Null-Object
}
@Override
public void addPlotsBeforeScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
// nothing to do; this is a Null-Object
}
@Override
public void addPlotsAfterScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
// nothing to do; this is a Null-Object
}
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, final long fromEpochMilli,
final long toEpochMilli) {
return new NullCustomAggregator();
}
}

View File

@@ -1,16 +0,0 @@
package org.lucares.pdb.plot.api;
public class NullCustomAggregator implements CustomAggregator {
@Override
public void addValue(boolean valueIsInYRange,final long epochMilli, final long value) {
// nothing to do; this is a null-object
}
@Override
public AggregatedData getAggregatedData() {
return null;
}
}

View File

@@ -2,45 +2,43 @@ package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.LineStyle;
public class ParallelRequestsAggregate implements AggregateHandler {
@Override
public void addGnuplotDefinitions(final StringBuilder result, final String separator,
final Collection<DataSeries> dataSeries) {
appendln(result, "set y2label \"Parallel Requests\"");
appendln(result, "set y2tics");
}
@Override
public void addGnuplotDefinitions(final StringBuilder result, final String separator,
final Collection<DataSeries> dataSeries) {
appendln(result, "set y2label \"Parallel Requests\"");
appendln(result, "set y2tics");
}
@Override
public void addPlotsBeforeScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
for (final DataSeries dataSerie : dataSeries) {
final AggregatedData aggregatedData = dataSerie.getAggregatedData();
if (aggregatedData != null) {
appendfln(result, "'%s' using 1:2 notitle with filledcurve axes x1y2 lw 1 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
dataSerie.getStyle().brighter().asGnuplotLineStyle()//
);
}
}
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with filledcurve axes x1y2 lw 1 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
lineStyle.brighter().asGnuplotLineStyle()//
);
}
@Override
public void addPlotsAfterScatter(final StringBuilder result, final Collection<DataSeries> dataSeries) {
// nothing to do: the parallel line plots shall be drawn below the scatter plot,
// so that you can see both. Therefore it must be drawn first.
}
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
if ((toEpochMilli - fromEpochMilli) <= TimeUnit.HOURS.toMillis(50)) {
return new ParallelRequestsAggregator(tmpDir, fromEpochMilli, toEpochMilli);
} else {
return null;
}
}
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, final long fromEpochMilli,
final long toEpochMilli) {
if ((toEpochMilli - fromEpochMilli) <= TimeUnit.HOURS.toMillis(5)) {
return new ParallelRequestsAggregator(tmpDir, fromEpochMilli, toEpochMilli);
} else {
return new NullCustomAggregator();
}
}
@Override
public Aggregate getAggregateType() {
return Aggregate.PARALLEL;
}
}

View File

@@ -93,4 +93,9 @@ public class ParallelRequestsAggregator implements CustomAggregator {
builder.append(NEWLINE);
}
@Override
public Aggregate getType() {
return Aggregate.PARALLEL;
}
}

View File

@@ -34,7 +34,7 @@ public class PlotSettings {
private AxisScale yAxisScale;
private AggregateHandler aggregate;
private AggregateHandlerCollection aggregates;
private int yRangeMin;
private int yRangeMax;
@@ -141,17 +141,17 @@ public class PlotSettings {
return "PlotSettings [query=" + query + ", height=" + height + ", width=" + width + ", thumbnailMaxWidth="
+ thumbnailMaxWidth + ", thumbnailMaxHeight=" + thumbnailMaxHeight + ", groupBy=" + groupBy
+ ", limitBy=" + limitBy + ", limit=" + limit + ", dateRangeAsString=" + dateRangeAsString
+ ", yAxisScale=" + yAxisScale + ", aggregate=" + aggregate + ", yRangeMin=" + yRangeMin
+ ", yAxisScale=" + yAxisScale + ", aggregates=" + aggregates + ", yRangeMin=" + yRangeMin
+ ", yRangeMax=" + yRangeMax + ", yRangeUnit=" + yRangeUnit + ", keyOutside=" + keyOutside
+ ", generateThumbnail=" + generateThumbnail + "]";
}
public void setAggregate(final AggregateHandler aggregate) {
this.aggregate = aggregate;
public void setAggregates(final AggregateHandlerCollection aggregates) {
this.aggregates = aggregates;
}
public AggregateHandler getAggregate() {
return aggregate;
public AggregateHandlerCollection getAggregates() {
return aggregates;
}
public void setKeyOutside(final boolean keyOutside) {

View File

@@ -2,9 +2,11 @@ package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotLineType;
import org.lucares.recommind.logs.LineStyle;
public class ScatterAggregateHandler implements AggregateHandler {
@@ -13,29 +15,16 @@ public class ScatterAggregateHandler implements AggregateHandler {
// TODO Auto-generated method stub
}
@Override
public void addPlotsBeforeScatter(StringBuilder result, Collection<DataSeries> dataSeries) {
for (final DataSeries dataSerie : dataSeries) {
final AggregatedData aggregatedData = dataSerie.getAggregatedData();
if (aggregatedData != null) {
appendfln(result, "'%s' using 1:2 title '%s' with %s %s, \\", //
aggregatedData.getDataFile(), //
dataSerie.getTitle(), //
GnuplotLineType.Points,
dataSerie.getStyle()//
);
}
}
}
@Override
public void addPlotsAfterScatter(StringBuilder result, Collection<DataSeries> dataSeries) {
// TODO Auto-generated method stub
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with %s %s, \\", //
aggregatedData.getDataFile(), //
gnuplotTitle(title), //
GnuplotLineType.Points, //
lineStyle//
);
}
@Override
@@ -44,4 +33,9 @@ public class ScatterAggregateHandler implements AggregateHandler {
return new ScatterAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli);
}
@Override
public Aggregate getAggregateType() {
return Aggregate.SCATTER;
}
}

View File

@@ -91,4 +91,9 @@ public class ScatterAggregator implements CustomAggregator {
return new AggregatedData("scatter", dataFile);
}
@Override
public Aggregate getType() {
return Aggregate.SCATTER;
}
}

View File

@@ -1,16 +1,16 @@
package org.lucares.recommind.logs;
import org.lucares.pdb.plot.api.AggregatedData;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
class CsvSummary {
private final int values;
private final long maxValue;
private final AggregatedData aggregatedData;
private final AggregatedDataCollection aggregatedData;
private final double statsAverage;
private final int plottedValues;
public CsvSummary(final int values, final int plottedValues, final long maxValue,
final double statsAverage, final AggregatedData aggregatedData) {
final double statsAverage, final AggregatedDataCollection aggregatedData) {
super();
this.values = values;
this.plottedValues = plottedValues;
@@ -48,7 +48,7 @@ class CsvSummary {
return statsAverage;
}
public AggregatedData getAggregatedData() {
public AggregatedDataCollection getAggregatedData() {
return aggregatedData;
}
}

View File

@@ -6,7 +6,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.plot.api.AggregatedData;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
import org.lucares.pdb.plot.api.Limit;
public interface DataSeries {
@@ -41,7 +41,7 @@ public interface DataSeries {
public LineStyle getStyle();
public AggregatedData getAggregatedData();
public AggregatedDataCollection getAggregatedData();
public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
final Map<String, Integer> result = new LinkedHashMap<>();

View File

@@ -1,6 +1,6 @@
package org.lucares.recommind.logs;
import org.lucares.pdb.plot.api.AggregatedData;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
public class FileBackedDataSeries implements DataSeries {
@@ -66,7 +66,7 @@ public class FileBackedDataSeries implements DataSeries {
}
@Override
public AggregatedData getAggregatedData() {
public AggregatedDataCollection getAggregatedData() {
return csvSummary.getAggregatedData();
}
}

View File

@@ -27,7 +27,7 @@ public class GnuplotFileGenerator {
appendfln(result, "set datafile separator \"%s\"", settings.getDatafileSeparator());
settings.getAggregate().addGnuplotDefinitions(result, settings.getDatafileSeparator(), dataSeries);
settings.getAggregates().addGnuplotDefinitions(result, settings.getDatafileSeparator(), dataSeries);
appendfln(result, "set timefmt '%s'", settings.getTimefmt());
@@ -103,10 +103,8 @@ public class GnuplotFileGenerator {
appendf(result, "plot ");
settings.getAggregate().addPlotsBeforeScatter(result, dataSeries);
settings.getAggregates().addPlots(result, dataSeries);
settings.getAggregate().addPlotsAfterScatter(result, dataSeries);
return result.toString();
}

View File

@@ -2,7 +2,7 @@ package org.lucares.recommind.logs;
import java.nio.file.Path;
import org.lucares.pdb.plot.api.AggregateHandler;
import org.lucares.pdb.plot.api.AggregateHandlerCollection;
import org.lucares.pdb.plot.api.AxisScale;
public class GnuplotSettings {
@@ -26,7 +26,7 @@ public class GnuplotSettings {
private final Path output;
private AxisScale yAxisScale;
private AggregateHandler aggregate;
private AggregateHandlerCollection aggregates;
private boolean keyOutside = false;
private XAxisSettings xAxisSettings = new XAxisSettings();
@@ -98,12 +98,12 @@ public class GnuplotSettings {
return yAxisScale;
}
public void setAggregate(final AggregateHandler aggregate) {
this.aggregate = aggregate;
public void setAggregates(final AggregateHandlerCollection aggregates) {
this.aggregates = aggregates;
}
public AggregateHandler getAggregate() {
return aggregate;
public AggregateHandlerCollection getAggregates() {
return aggregates;
}
public void setKeyOutside(final boolean keyOutside) {

View File

@@ -22,7 +22,7 @@ import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Query;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.AggregatorCollection;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.TimeRangeUnitInternal;
@@ -115,7 +115,7 @@ public class Plotter {
defineXAxis(gnuplotSettings, plotSettings.dateRange());
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
gnuplotSettings.setAggregate(plotSettings.getAggregate());
gnuplotSettings.setAggregates(plotSettings.getAggregates());
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
plotSettings.getYRangeUnit());
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
@@ -132,7 +132,7 @@ public class Plotter {
defineXAxis(gnuplotSettings, plotSettings.dateRange());
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
gnuplotSettings.setAggregate(plotSettings.getAggregate());
gnuplotSettings.setAggregates(plotSettings.getAggregates());
defineYRange(gnuplotSettings, plotSettings.getYRangeMin(), plotSettings.getYRangeMax(),
plotSettings.getYRangeUnit());
gnuplotSettings.setKeyOutside(false);
@@ -205,7 +205,7 @@ public class Plotter {
final long maxValue = plotSettings.getYRangeUnit() == TimeRangeUnitInternal.AUTOMATIC ? Long.MAX_VALUE
: plotSettings.getYRangeUnit().toMilliSeconds(plotSettings.getYRangeMax());
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(tmpDir, plotSettings, fromEpochMilli,
final AggregatorCollection aggregator = plotSettings.getAggregates().createCustomAggregator(tmpDir, plotSettings, fromEpochMilli,
toEpochMilli);
int count = 0; // number of values in the x-axis range (used to compute stats)
@@ -228,8 +228,6 @@ public class Plotter {
final long value = entry.get(i + 1);
// compute stats
count++;
statsMaxValue = Math.max(statsMaxValue, value);
@@ -293,7 +291,5 @@ public class Plotter {
result.append(")");
return result.toString();
}
}