diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java b/pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java index bf187b8..60817ed 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java @@ -33,4 +33,9 @@ public class GroupResult { return result; } + + @Override + public String toString() { + return groupedBy.toString(); + } } diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/Result.java b/pdb-api/src/main/java/org/lucares/pdb/api/Result.java index a774839..3299e56 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/Result.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/Result.java @@ -27,4 +27,9 @@ public class Result { public List getGroups() { return new ArrayList<>(groupResults); } + + @Override + public String toString() { + return groupResults.toString(); + } } diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java index 1ca29af..a91bb04 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java @@ -14,11 +14,13 @@ import org.lucares.utils.byteencoder.VariableByteEncoder; public class Tags implements Comparable { + private static final String DEFAULT_GROUP = ""; public static StringCompressor STRING_COMPRESSOR = null; private static final byte[] EMPTY_BYTES = new byte[0]; public static final Tags EMPTY = new Tags(); private final List tags; + private int hashCode = 0; public Tags() { tags = new ArrayList<>(); @@ -216,10 +218,12 @@ public class Tags implements Comparable { @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((tags == null) ? 0 : tags.hashCode()); - return result; + if (hashCode == 0) { + final int prime = 31; + final int result = 1; + hashCode = prime * result + ((tags == null) ? 0 : tags.hashCode()); + } + return hashCode; } @Override @@ -278,4 +282,19 @@ public class Tags implements Comparable { return result.toString(); } + + public String asValueString() { + final StringBuilder result = new StringBuilder(); + if (isEmpty()) { + result.append(DEFAULT_GROUP); + } else { + forEach((k, v) -> { + if (result.length() > 0) { + result.append(" / "); + } + result.append(v); + }); + } + return result.toString(); + } } diff --git a/pdb-js/src/app/plot.service.ts b/pdb-js/src/app/plot.service.ts index bc726af..d2f7134 100644 --- a/pdb-js/src/app/plot.service.ts +++ b/pdb-js/src/app/plot.service.ts @@ -17,6 +17,7 @@ export class PlotService { this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration)); this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount)); this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count)); + this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", true, DataType.Group, DataType.Count)); this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration)); @@ -25,7 +26,6 @@ export class PlotService { this.plotTypes.push(new PlotType("VIOLIN", "Violin", "violin-chart", false, DataType.Group, DataType.Duration)); this.plotTypes.push(new PlotType("STRIP", "Strip", "strip-chart", false, DataType.Group, DataType.Duration)); this.plotTypes.push(new PlotType("PIE", "Pie", "pie-chart", false, DataType.Other, DataType.Other)); - this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("STEP_FIT", "Step Fit", "step-fit", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("LAG", "Lag", "lag-plot", false, DataType.Other, DataType.Other)); this.plotTypes.push(new PlotType("ACF", "ACF", "acf-plot", false, DataType.Other, DataType.Other)); diff --git a/pdb-js/src/assets/js/invaders.js b/pdb-js/src/assets/js/invaders.js index d543bd4..3783d72 100644 --- a/pdb-js/src/assets/js/invaders.js +++ b/pdb-js/src/assets/js/invaders.js @@ -22,11 +22,13 @@ function initInvaders(parentDivId) { var parent = $('#'+parentDivId); var height = parent.height(); var width = parent.width(); - $('body').append('
Points: 0
Kills: 0
Game Over
'); + $('body').append('
Points: 0
Kills: 0
Game Over
'); $('#'+invaders_parentDivId).offset({ top: parent.offset().top, left: parent.offset().left}) $('#'+invaders_parentDivId).height(parent.height()); $('#'+invaders_parentDivId).width(parent.width()); + $('#'+invaders_parentDivId) + $('.invader_notify').click(function() { // restart the game $('#'+invaders_parentDivId).remove(); @@ -81,7 +83,7 @@ function addInvader() var top = 10; // start at the top var left = invaders_margin + random(width-2*invaders_margin); // spread horizontally, but keep margin to left and right - parent.append('
'); + parent.append(''); $("#"+id).click(function() { diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java index b9cc994..16ceebd 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java @@ -5,6 +5,8 @@ package org.lucares.pdb.plot.api; * drawn. */ public enum Aggregate { + BAR, + PARALLEL, SCATTER, @@ -16,5 +18,6 @@ public enum Aggregate { */ CUM_DISTRIBUTION, - HISTOGRAM + HISTOGRAM, + } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java index 3636113..e075f0a 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java @@ -52,7 +52,7 @@ public abstract class AggregateHandler implements Appender { abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection dataSeries); - abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, + abstract void addPlot(StringBuilder result, CustomAggregator aggregator, LineStyle lineStyle, Optional title); abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java index 37084e1..d644746 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java @@ -22,7 +22,7 @@ public class AggregateHandlerCollection { private final List aggregateHandlers = new ArrayList<>(); - public void add(final AggregateHandler aggregateHandler) { + public void addAggregateHandler(final AggregateHandler aggregateHandler) { aggregateHandlers.add(aggregateHandler); } @@ -99,14 +99,21 @@ public class AggregateHandlerCollection { final List handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers, PLOTTING_ORDER); for (final AggregateHandler handler : handlersInPlottingOrder) { + long index = 0; for (final DataSeries dataSerie : dataSeries) { final Optional title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty(); - final Optional aggregatedData = dataSerie.getAggregatedData() - .get(handler.getAggregateType()); - if (aggregatedData.isPresent()) { - handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title); + final Optional optionalAggregator = dataSerie.getAggregators() + .getCustomAggregator(handler.getAggregateType()); + if (optionalAggregator.isPresent()) { + final CustomAggregator aggregator = optionalAggregator.get(); + if (aggregator instanceof IndexedAggregator) { + ((IndexedAggregator) aggregator).setIndex(index); + index++; + } + + handler.addPlot(result, aggregator, dataSerie.getStyle(), title); } } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java index 9cdbbf3..a208a7d 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java @@ -4,6 +4,8 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Optional; +// TODO remove +@Deprecated public class AggregatedDataCollection implements Iterable { private final LinkedHashMap aggregatedDatas = new LinkedHashMap<>(); diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java index 52f347e..66c5b26 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java @@ -1,29 +1,30 @@ package org.lucares.pdb.plot.api; -import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.lucares.pdb.api.Tags; public class AggregatorCollection { - private final List aggregators; + private final Map aggregators = new LinkedHashMap<>(); public AggregatorCollection(final List aggregators) { - this.aggregators = aggregators; + + for (final CustomAggregator aggregator : aggregators) { + this.aggregators.put(aggregator.getType(), aggregator); + } } - public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { - for (final CustomAggregator aggregator : aggregators) { + public void addValue(final Tags groupedBy, final boolean valueIsInYRange, final long epochMilli, final long value) { + for (final CustomAggregator aggregator : aggregators.values()) { aggregator.addValue(valueIsInYRange, epochMilli, value); } } - public AggregatedDataCollection getAggregatedData() throws IOException { - - final AggregatedDataCollection result = new AggregatedDataCollection(); - - for (final CustomAggregator aggregator : aggregators) { - result.put(aggregator.getType(), aggregator.getAggregatedData()); - } - - return result; + public Optional getCustomAggregator(final Aggregate type) { + return Optional.ofNullable(aggregators.get(type)); } + } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java index c857b1d..9ab0a44 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java @@ -14,4 +14,5 @@ public interface Appender { default void appendf(final StringBuilder builder, final String format, final Object... args) { builder.append(String.format(Locale.US, format, args)); } + } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartAggregator.java new file mode 100644 index 0000000..73992a5 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartAggregator.java @@ -0,0 +1,69 @@ +package org.lucares.pdb.plot.api; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.lucares.pdb.api.RuntimeIOException; + +public class BarChartAggregator implements CustomAggregator, IndexedAggregator { + + long count = 0; + + private final Path tmpDir; + + private Long index = null; + + public BarChartAggregator(final Path tmpDir) { + super(); + this.tmpDir = tmpDir; + } + + @Override + public void setIndex(final long index) { + this.index = index; + } + + @Override + public long getIndex() throws IllegalStateException { + if (this.index == null) { + throw new IllegalStateException("index was not set"); + } + return this.index; + } + + @Override + public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { + count++; + } + + @Override + public AggregatedData getAggregatedData() { + try { + final File dataFile = File.createTempFile("bar", ".dat", tmpDir.toFile()); + + final StringBuilder csv = new StringBuilder(); + + csv.append(index + 0.5); + csv.append(","); + csv.append(""); + csv.append(","); + csv.append(count); + csv.append("\n"); + + Files.writeString(dataFile.toPath(), csv.toString(), StandardCharsets.UTF_8); + final AggregatedData result = new AggregatedData("label", dataFile); + return result; + } catch (final IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + public Aggregate getType() { + return Aggregate.BAR; + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartHandler.java new file mode 100644 index 0000000..2b5e683 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartHandler.java @@ -0,0 +1,97 @@ +package org.lucares.pdb.plot.api; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import org.lucares.recommind.logs.AxisSettings; +import org.lucares.recommind.logs.AxisSettings.Type; +import org.lucares.recommind.logs.DataSeries; +import org.lucares.recommind.logs.GnuplotAxis; +import org.lucares.recommind.logs.GnuplotLineType; +import org.lucares.recommind.logs.GnuplotSettings; +import org.lucares.recommind.logs.LineStyle; + +public class BarChartHandler extends AggregateHandler { + + @Override + Type getAxisType(final GnuplotAxis axis) { + switch (axis) { + case X1: + case X2: + return Type.Group; + case Y1: + case Y2: + return Type.Number; + default: + throw new IllegalArgumentException("Unexpected value: " + axis); + } + } + + @Override + Aggregate getAggregateType() { + return Aggregate.BAR; + } + + @Override + AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = new AxisSettings(); + result.setLabel("Bar"); + result.setType(Type.Group); + result.setAxis(getxAxis()); + result.setTicsEnabled(true); + result.setFrom("0"); + result.setTo(String.valueOf(dataSeries.size())); + + final List ticsLabels = new ArrayList<>(); + int index = 1; + for (final DataSeries series : dataSeries) { + ticsLabels.add("\"" + series.getTitle() + "\" " + index); + index++; + } + // result.setTics(ticsLabels); + return result; + } + + @Override + AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = new AxisSettings(); + result.setLabel("Count"); + result.setType(Type.Number); + result.setAxis(getyAxis()); + result.setTicsEnabled(true); + result.setFrom("0"); + result.setLogscale(settings.getYAxisScale() == AxisScale.LOG10); + return result; + } + + @Override + void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle, + final Optional title) { + + final AggregatedData aggregatedData = aggregator.getAggregatedData(); + + /* + * appendfln(result, + * "'%s' using 1:3:xtic(2) notitle with %s axes %s fs solid %s, \\", // + * aggregatedData.getDataFile(), // GnuplotLineType.Bar, // gnuplotXYAxis(), // + * lineStyle// ); + */ + appendfln(result, "'%s' using 1:3:xtic(2) %s with %s axes %s fs solid %s, \\", // + aggregatedData.getDataFile(), // + gnuplotTitle(title), // + GnuplotLineType.Bar, // + gnuplotXYAxis(), // + lineStyle.brighter()// + ); + } + + @Override + CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings, + final long fromEpochMilli, final long toEpochMilli) { + return new BarChartAggregator(tmpDir); + } + +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java index f00ff7f..56c77e3 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java @@ -12,6 +12,7 @@ import java.util.LinkedHashMap; import org.lucares.collections.LongLongConsumer; import org.lucares.collections.LongLongHashMap; +import org.lucares.pdb.api.RuntimeIOException; public class CumulativeDistributionCustomAggregator implements CustomAggregator { @@ -78,46 +79,50 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator } @Override - public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) { + public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { map.compute(value, 0, l -> l + 1); totalValues++; } @Override - public AggregatedData getAggregatedData() throws IOException { - final char separator = ','; - final char newline = '\n'; + public AggregatedData getAggregatedData() { + try { + final char separator = ','; + final char newline = '\n'; - final ToPercentiles toPercentiles = new ToPercentiles(totalValues); - map.forEachOrdered(toPercentiles); + final ToPercentiles toPercentiles = new ToPercentiles(totalValues); + map.forEachOrdered(toPercentiles); - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - try (final Writer output = new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { - final StringBuilder data = new StringBuilder(); - if (map.size() > 0) { - // compute the percentiles - toPercentiles.getPercentiles().forEach((percentile, value) -> { + final StringBuilder data = new StringBuilder(); + if (map.size() > 0) { + // compute the percentiles + toPercentiles.getPercentiles().forEach((percentile, value) -> { - data.append(percentile); + data.append(percentile); + data.append(separator); + data.append(value); + data.append(newline); + }); + + final long maxValue = toPercentiles.getMaxValue(); + data.append(100); data.append(separator); - data.append(value); + data.append(maxValue); data.append(newline); - }); + } + output.write(data.toString()); - final long maxValue = toPercentiles.getMaxValue(); - data.append(100); - data.append(separator); - data.append(maxValue); - data.append(newline); } - output.write(data.toString()); + final String title = String.format("cumulative distribution"); + return new AggregatedData(title, dataFile); + } catch (final IOException e) { + throw new RuntimeIOException(e); } - - final String title = String.format("cumulative distribution"); - return new AggregatedData(title, dataFile); } @Override diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java index 20417dd..e8d8794 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java @@ -4,18 +4,18 @@ import java.nio.file.Path; import java.util.Collection; import java.util.Optional; +import org.lucares.recommind.logs.AxisSettings; +import org.lucares.recommind.logs.AxisSettings.Type; import org.lucares.recommind.logs.AxisTime; import org.lucares.recommind.logs.DataSeries; import org.lucares.recommind.logs.GnuplotAxis; import org.lucares.recommind.logs.GnuplotSettings; import org.lucares.recommind.logs.LineStyle; -import org.lucares.recommind.logs.AxisSettings; -import org.lucares.recommind.logs.AxisSettings.Type; public class CumulativeDistributionHandler extends AggregateHandler { @Override - public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, + public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings, final long fromEpochMilli, final long toEpochMilli) { return new CumulativeDistributionCustomAggregator(tmpDir); } @@ -24,7 +24,7 @@ public class CumulativeDistributionHandler extends AggregateHandler { } @Override - Type getAxisType(GnuplotAxis axis) { + Type getAxisType(final GnuplotAxis axis) { switch (axis) { case X1: case X2: @@ -38,15 +38,15 @@ public class CumulativeDistributionHandler extends AggregateHandler { } @Override - public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection dataSeries) { - AxisSettings result = AxisTime.createYAxis(settings, dataSeries); + public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = AxisTime.createYAxis(settings, dataSeries); result.setAxis(getyAxis()); return result; } @Override - public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection dataSeries) { - AxisSettings result = new AxisSettings(); + public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { + final AxisSettings result = new AxisSettings(); result.setLabel("Cumulative Distribution"); result.setType(Type.Percent); result.setAxis(getxAxis()); @@ -57,8 +57,8 @@ public class CumulativeDistributionHandler extends AggregateHandler { return result; } - private int computeTicIncrement(GnuplotSettings settings) { - int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE; + private int computeTicIncrement(final GnuplotSettings settings) { + final int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE; if (widthByFontSize < 50) { return 20; } else if (widthByFontSize < 75) { @@ -69,8 +69,11 @@ public class CumulativeDistributionHandler extends AggregateHandler { } @Override - public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, - Optional title) { + public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle, + final Optional title) { + + final AggregatedData aggregatedData = aggregator.getAggregatedData(); + appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", // aggregatedData.getDataFile().getAbsolutePath(), // gnuplotTitle(title), // diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java index 886b670..0c74448 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java @@ -1,12 +1,10 @@ package org.lucares.pdb.plot.api; -import java.io.IOException; - public interface CustomAggregator { void addValue(boolean valueIsInYRange, long epochMilli, long value); - AggregatedData getAggregatedData() throws IOException; + AggregatedData getAggregatedData(); Aggregate getType(); } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomGroupedAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomGroupedAggregator.java new file mode 100644 index 0000000..d041882 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomGroupedAggregator.java @@ -0,0 +1,14 @@ +//package org.lucares.pdb.plot.api; +// +//import java.io.IOException; +//import java.util.Optional; +// +//import org.lucares.pdb.api.Tags; +// +//public interface CustomGroupedAggregator { +// void addValue(Tags groupedBy, boolean valueIsInYRange, long epochMilli, long value); +// +// Optional getAggregatedData() throws IOException; +// +// Aggregate getType(); +//} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java index e773214..a7748d9 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java @@ -11,6 +11,7 @@ import java.nio.file.Path; import org.lucares.collections.LongLongConsumer; import org.lucares.collections.LongLongHashMap; +import org.lucares.pdb.api.RuntimeIOException; public class HistogramAggregator implements CustomAggregator { @@ -57,40 +58,44 @@ public class HistogramAggregator implements CustomAggregator { } @Override - public AggregatedData getAggregatedData() throws IOException { - final char separator = ','; - final char newline = '\n'; + public AggregatedData getAggregatedData() { + try { + final char separator = ','; + final char newline = '\n'; - final int numBins = plotSettings.getWidth() / 8; - final int binWidth = Math.max((int) (max) / numBins, 1); + final int numBins = plotSettings.getWidth() / 8; + final int binWidth = Math.max((int) (max) / numBins, 1); - final ToBins toBins = new ToBins(numBins, binWidth); - map.forEachOrdered(toBins); + final ToBins toBins = new ToBins(numBins, binWidth); + map.forEachOrdered(toBins); - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - try (final Writer output = new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { - final StringBuilder data = new StringBuilder(); - if (map.size() > 0) { - // compute the percentiles - final LongLongHashMap bins = toBins.getBins(); - for (int i = 0; i < numBins; i++) { - final int bin = i * binWidth; - final long value = bins.get(bin, 0); - data.append(bin); - data.append(separator); - data.append(value); - data.append(newline); + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final StringBuilder data = new StringBuilder(); + if (map.size() > 0) { + // compute the percentiles + final LongLongHashMap bins = toBins.getBins(); + for (int i = 0; i < numBins; i++) { + final int bin = i * binWidth; + final long value = bins.get(bin, 0); + data.append(bin); + data.append(separator); + data.append(value); + data.append(newline); + + } } + output.write(data.toString()); + System.out.println(data.toString()); } - output.write(data.toString()); - System.out.println(data.toString()); - + final AggregatedData result = new AggregatedData("histogram", dataFile); + return result; + } catch (final IOException e) { + throw new RuntimeIOException(e); } - final AggregatedData result = new AggregatedData("histogram", dataFile); - return result; } @Override diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java index 364bfbf..36637e0 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java @@ -54,8 +54,10 @@ public class HistogramHandler extends AggregateHandler { } @Override - void addPlot(final StringBuilder result, final AggregatedData aggregatedData, final LineStyle lineStyle, + void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle, final Optional title) { + final AggregatedData aggregatedData = aggregator.getAggregatedData(); + appendfln(result, "'%s' using 1:2 %s with boxes axes %s lw 1 %s, \\", // aggregatedData.getDataFile().getAbsolutePath(), // gnuplotTitle(title), // diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/IndexedAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/IndexedAggregator.java new file mode 100644 index 0000000..188f309 --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/IndexedAggregator.java @@ -0,0 +1,16 @@ +package org.lucares.pdb.plot.api; + +public interface IndexedAggregator { + + /** + * Set the index of this {@link CustomAggregator}. + */ + public void setIndex(long index); + + /** + * Returns the index. + * + * @throws IllegalStateException if the index was no set + */ + public long getIndex() throws IllegalStateException; +} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ObsoleteBarChartAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ObsoleteBarChartAggregator.java new file mode 100644 index 0000000..e077b6e --- /dev/null +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ObsoleteBarChartAggregator.java @@ -0,0 +1,66 @@ +//package org.lucares.pdb.plot.api; +// +//import java.io.File; +//import java.io.IOException; +//import java.nio.charset.StandardCharsets; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.util.Map; +//import java.util.Map.Entry; +//import java.util.Optional; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.atomic.AtomicLong; +// +//import org.lucares.pdb.api.Tags; +// +//public class ObsoleteBarChartAggregator implements CustomGroupedAggregator { +// +// Map counts = new ConcurrentHashMap<>(); +// +// private final Path tmpDir; +// +// public ObsoleteBarChartAggregator(final Path tmpDir) { +// super(); +// this.tmpDir = tmpDir; +// } +// +// @Override +// public void addValue(final Tags groupedBy, final boolean valueIsInYRange, final long epochMilli, final long value) { +// final AtomicLong val = counts.computeIfAbsent(groupedBy, (k) -> new AtomicLong()); +// val.incrementAndGet(); +// } +// +// @Override +// public Optional getAggregatedData() throws IOException { +// +// if (counts.entrySet().isEmpty()) { +// return Optional.empty(); +// } +// +// final File dataFile = File.createTempFile("bar", ".dat", tmpDir.toFile()); +// +// final StringBuilder csv = new StringBuilder(); +// +// int position = 1; +// for (final Entry e : counts.entrySet()) { +// csv.append(position); +// csv.append(","); +// csv.append(e.getKey().asValueString()); +// csv.append(","); +// csv.append(e.getValue().get()); +// csv.append("\n"); +// +// position++; +// } +// +// Files.writeString(dataFile.toPath(), csv.toString(), StandardCharsets.UTF_8); +// final AggregatedData result = new AggregatedData("label", dataFile); +// return Optional.of(result); +// } +// +// @Override +// public Aggregate getType() { +// return Aggregate.BAR; +// } +// +//} diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregate.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregate.java index 7ab4b15..3c4e8f2 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregate.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregate.java @@ -5,18 +5,18 @@ import java.util.Collection; import java.util.Optional; import java.util.concurrent.TimeUnit; -import org.lucares.recommind.logs.GnuplotAxis; -import org.lucares.recommind.logs.GnuplotSettings; -import org.lucares.recommind.logs.LineStyle; import org.lucares.recommind.logs.AxisSettings; import org.lucares.recommind.logs.AxisSettings.Type; import org.lucares.recommind.logs.AxisTime; import org.lucares.recommind.logs.DataSeries; +import org.lucares.recommind.logs.GnuplotAxis; +import org.lucares.recommind.logs.GnuplotSettings; +import org.lucares.recommind.logs.LineStyle; public class ParallelRequestsAggregate extends AggregateHandler { @Override - Type getAxisType(GnuplotAxis axis) { + Type getAxisType(final GnuplotAxis axis) { switch (axis) { case X1: case X2: @@ -30,7 +30,7 @@ public class ParallelRequestsAggregate extends AggregateHandler { } @Override - public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection dataSeries) { + public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { final AxisSettings result = new AxisSettings(); result.setLabel("Parallel Requests"); result.setType(Type.Number); @@ -41,15 +41,16 @@ public class ParallelRequestsAggregate extends AggregateHandler { } @Override - public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection dataSeries) { + public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { final AxisSettings result = AxisTime.createXAxis(settings); result.setAxis(getxAxis()); return result; } @Override - public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, - Optional title) { + public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle, + final Optional title) { + final AggregatedData aggregatedData = aggregator.getAggregatedData(); appendfln(result, "'%s' using 1:2 %s with filledcurve axes %s lw 1 %s, \\", // aggregatedData.getDataFile().getAbsolutePath(), // gnuplotTitle(title), // @@ -59,7 +60,7 @@ public class ParallelRequestsAggregate extends AggregateHandler { } @Override - public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, + public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings, final long fromEpochMilli, final long toEpochMilli) { if ((toEpochMilli - fromEpochMilli) <= TimeUnit.HOURS.toMillis(50)) { return new ParallelRequestsAggregator(tmpDir, fromEpochMilli, toEpochMilli); diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregator.java index 47b8648..b822707 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregator.java @@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Locale; +import org.lucares.pdb.api.RuntimeIOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,7 +41,7 @@ public class ParallelRequestsAggregator implements CustomAggregator { } @Override - public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) { + public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { final int endPos = (int) (epochMilli - fromEpochMilli); increments[endPos]--; @@ -51,40 +52,43 @@ public class ParallelRequestsAggregator implements CustomAggregator { } @Override - public AggregatedData getAggregatedData() throws IOException { + public AggregatedData getAggregatedData() { + try { + final long start = System.nanoTime(); - final long start = System.nanoTime(); + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - try (final Writer output = new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final StringBuilder data = new StringBuilder(); - final StringBuilder data = new StringBuilder(); + // first and last value should be 0, or gnuplot will draw a diagonal line + appendTimeAndValue(data, fromEpochMilli, 0); - // first and last value should be 0, or gnuplot will draw a diagonal line - appendTimeAndValue(data, fromEpochMilli, 0); - - int value = 0; - for (int i = 0; i < increments.length - 1; i++) { - final int increment = increments[i]; - final int nextIncrement = increments[i + 1]; - if (increment != 0 || nextIncrement != 0) { - value += increment; - appendTimeAndValue(data, fromEpochMilli + i, value); + int value = 0; + for (int i = 0; i < increments.length - 1; i++) { + final int increment = increments[i]; + final int nextIncrement = increments[i + 1]; + if (increment != 0 || nextIncrement != 0) { + value += increment; + appendTimeAndValue(data, fromEpochMilli + i, value); + } } + + // first and last value should be 0, or gnuplot will draw a diagonal line + appendTimeAndValue(data, toEpochMilli, 0); + + output.write(data.toString()); + } - // first and last value should be 0, or gnuplot will draw a diagonal line - appendTimeAndValue(data, toEpochMilli, 0); - - output.write(data.toString()); - + final String title = String.format("parallelRequests"); + METRICS_LOGGER.debug("wrote parallelRequests csv in: {}ms file={}", + (System.nanoTime() - start) / 1_000_000.0, dataFile); + return new AggregatedData(title, dataFile); + } catch (final IOException e) { + throw new RuntimeIOException(e); } - - final String title = String.format("parallelRequests"); - METRICS_LOGGER.debug("wrote parallelRequests csv in: {}ms file={}", (System.nanoTime() - start) / 1_000_000.0, - dataFile); - return new AggregatedData(title, dataFile); } private void appendTimeAndValue(final StringBuilder builder, final long timeEpochMilli, final int value) { diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregateHandler.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregateHandler.java index 6126a58..4a827c5 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregateHandler.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregateHandler.java @@ -5,18 +5,18 @@ import java.util.Collection; import java.util.Optional; import org.lucares.recommind.logs.AxisSettings; +import org.lucares.recommind.logs.AxisSettings.Type; import org.lucares.recommind.logs.AxisTime; import org.lucares.recommind.logs.DataSeries; import org.lucares.recommind.logs.GnuplotAxis; import org.lucares.recommind.logs.GnuplotLineType; import org.lucares.recommind.logs.GnuplotSettings; import org.lucares.recommind.logs.LineStyle; -import org.lucares.recommind.logs.AxisSettings.Type; public class ScatterAggregateHandler extends AggregateHandler { @Override - Type getAxisType(GnuplotAxis axis) { + Type getAxisType(final GnuplotAxis axis) { switch (axis) { case X1: case X2: @@ -30,23 +30,24 @@ public class ScatterAggregateHandler extends AggregateHandler { } @Override - public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection dataSeries) { + public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { final AxisSettings result = AxisTime.createYAxis(settings, dataSeries); result.setAxis(getyAxis()); return result; } @Override - public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection dataSeries) { + public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection dataSeries) { final AxisSettings result = AxisTime.createXAxis(settings); result.setAxis(getxAxis()); return result; } @Override - public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, - Optional title) { + public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle, + final Optional title) { + final AggregatedData aggregatedData = aggregator.getAggregatedData(); appendfln(result, "'%s' using 1:2 %s with %s axes %s %s, \\", // aggregatedData.getDataFile(), // gnuplotTitle(title), // @@ -57,8 +58,8 @@ public class ScatterAggregateHandler extends AggregateHandler { } @Override - public CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, - long toEpochMilli) { + public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings, + final long fromEpochMilli, final long toEpochMilli) { return new ScatterAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli); } diff --git a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregator.java b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregator.java index a4c4cbd..aa5434d 100644 --- a/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregator.java +++ b/pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregator.java @@ -12,6 +12,7 @@ import java.util.Locale; import java.util.concurrent.TimeUnit; import org.lucares.collections.Sparse2DLongArray; +import org.lucares.pdb.api.RuntimeIOException; import org.lucares.recommind.logs.GnuplotSettings; import org.lucares.recommind.logs.LambdaFriendlyWriter; import org.lucares.recommind.logs.LongUtils; @@ -29,9 +30,10 @@ public class ScatterAggregator implements CustomAggregator { private final long maxValue; private final long durationMillisPerPixel; - private Path tmpDir; + private final Path tmpDir; - public ScatterAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, long toEpochMilli) { + public ScatterAggregator(final Path tmpDir, final PlotSettings plotSettings, final long fromEpochMilli, + final long toEpochMilli) { this.tmpDir = tmpDir; useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); @@ -49,47 +51,50 @@ public class ScatterAggregator implements CustomAggregator { } @Override - public void addValue(boolean valueIsInYRange, long epochMilli, long value) { + public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) { final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel; final long roundedValue = value - value % durationMillisPerPixel; matrix2d.put(roundedEpochMilli, roundedValue, 1); } @Override - public AggregatedData getAggregatedData() throws IOException { + public AggregatedData getAggregatedData() { + try { + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); + final int separator = ','; + final int newline = '\n'; - final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - final int separator = ','; - final int newline = '\n'; + final long[] actualValuesWritten = new long[1]; + final StringBuilder formattedDateBuilder = new StringBuilder(); + try (final LambdaFriendlyWriter output = new LambdaFriendlyWriter(new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1))); + final Formatter formatter = new Formatter(formattedDateBuilder);) { - long[] actualValuesWritten = new long[1]; - final StringBuilder formattedDateBuilder = new StringBuilder(); - try (final LambdaFriendlyWriter output = new LambdaFriendlyWriter(new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1))); - final Formatter formatter = new Formatter(formattedDateBuilder);) { + matrix2d.forEach((epochMilli, value, __) -> { - matrix2d.forEach((epochMilli, value, __) -> { + final String stringValue = LongUtils.longToString(value); + final String formattedDate; - final String stringValue = LongUtils.longToString(value); - final String formattedDate; + if (useMillis) { + formattedDateBuilder.delete(0, formattedDateBuilder.length()); + formatter.format(Locale.US, "%.3f", epochMilli / 1000.0); + formattedDate = formattedDateBuilder.toString(); + } else { + formattedDate = String.valueOf(epochMilli / 1000); + } - if (useMillis) { - formattedDateBuilder.delete(0, formattedDateBuilder.length()); - formatter.format(Locale.US, "%.3f", epochMilli / 1000.0); - formattedDate = formattedDateBuilder.toString(); - } else { - formattedDate = String.valueOf(epochMilli / 1000); - } + output.write(formattedDate); + output.write(separator); + output.write(stringValue); + output.write(newline); + actualValuesWritten[0]++; + }); + } - output.write(formattedDate); - output.write(separator); - output.write(stringValue); - output.write(newline); - actualValuesWritten[0]++; - }); + return new AggregatedData("scatter", dataFile); + } catch (final IOException e) { + throw new RuntimeIOException(e); } - - return new AggregatedData("scatter", dataFile); } @Override diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java index 7ca9c4d..01e457d 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; public class AxisSettings { public enum Type { - Number, Time, Duration, Percent, HistogramBin, HistogramCount + Number, Time, Duration, Percent, HistogramBin, HistogramCount, Group } private String format = ""; diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java index e4195bd..41222d5 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java @@ -1,22 +1,22 @@ package org.lucares.recommind.logs; -import org.lucares.pdb.plot.api.AggregatedDataCollection; +import org.lucares.pdb.plot.api.AggregatorCollection; class CsvSummary { private final int values; private final long maxValue; - private final AggregatedDataCollection aggregatedData; + private final AggregatorCollection aggregators; private final double statsAverage; private final int plottedValues; public CsvSummary(final int values, final int plottedValues, final long maxValue, final double statsAverage, - final AggregatedDataCollection aggregatedData) { + final AggregatorCollection aggregators) { super(); this.values = values; this.plottedValues = plottedValues; this.maxValue = maxValue; this.statsAverage = statsAverage; - this.aggregatedData = aggregatedData; + this.aggregators = aggregators; } /** @@ -47,7 +47,7 @@ class CsvSummary { return statsAverage; } - public AggregatedDataCollection getAggregatedData() { - return aggregatedData; + public AggregatorCollection getAggregators() { + return aggregators; } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java index 1a77f07..3ad0bab 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java @@ -6,7 +6,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.lucares.pdb.plot.api.AggregatedDataCollection; +import org.lucares.pdb.plot.api.AggregatorCollection; import org.lucares.pdb.plot.api.Limit; public interface DataSeries { @@ -41,7 +41,7 @@ public interface DataSeries { public LineStyle getStyle(); - public AggregatedDataCollection getAggregatedData(); + public AggregatorCollection getAggregators(); public static Map toMap(final List dataSeries) { final Map result = new LinkedHashMap<>(); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java index 2f34d5b..7fc90ec 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java @@ -1,6 +1,6 @@ package org.lucares.recommind.logs; -import org.lucares.pdb.plot.api.AggregatedDataCollection; +import org.lucares.pdb.plot.api.AggregatorCollection; public class FileBackedDataSeries implements DataSeries { @@ -64,7 +64,7 @@ public class FileBackedDataSeries implements DataSeries { } @Override - public AggregatedDataCollection getAggregatedData() { - return csvSummary.getAggregatedData(); + public AggregatorCollection getAggregators() { + return csvSummary.getAggregators(); } } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java index a5859f1..8b7e1e7 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java @@ -37,9 +37,9 @@ public interface GnuplotColorPalettes { ); List GNUPLOT_REORDERED = Arrays.asList(// - GnuplotColor.byHex("0072b2"), // blue - GnuplotColor.byHex("e69f00"), // orange - GnuplotColor.byHex("9400D3"), // purple + GnuplotColor.byHex("0072b2"), // blue 00A2FF, 0091E6, 0072B2, 005180, 002840 + GnuplotColor.byHex("e69f00"), // orange FFAE00, E69F00, BF8300, 805700, 402C00 + GnuplotColor.byHex("9400D3"), // purple B300FF, A100E6, 9400D3, 590080, 2D0040 GnuplotColor.byHex("009e73"), // green GnuplotColor.byHex("f0e442"), // yellow GnuplotColor.byHex("e51e10"), // red diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java index f5f97bb..3decd37 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java @@ -64,6 +64,9 @@ public class GnuplotFileGenerator implements Appender { } } + // appendfln(result, "set xrange [-1:1]"); + appendfln(result, "set boxwidth 0.5"); + appendf(result, "plot "); settings.getAggregates().addPlots(result, dataSeries); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java index 2d7fd00..a5fb428 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java @@ -3,11 +3,13 @@ package org.lucares.recommind.logs; public enum GnuplotLineType { LINE("line"), + Bar("boxes"), + Points("points"); private String gnuplotLineType; - GnuplotLineType(String gnuplotLineType) { + GnuplotLineType(final String gnuplotLineType) { this.gnuplotLineType = gnuplotLineType; } diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java index 03d8ef6..78843d1 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -26,7 +26,6 @@ import org.lucares.pdb.plot.api.Limit; import org.lucares.pdb.plot.api.PlotSettings; import org.lucares.pdb.plot.api.TimeRangeUnitInternal; import org.lucares.performance.db.PerformanceDb; -import org.lucares.utils.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,8 +34,6 @@ public class Plotter { private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class); private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter"); - static final String DEFAULT_GROUP = ""; - private final PerformanceDb db; private final Path tmpBaseDir; private final Path outputDir; @@ -95,6 +92,7 @@ public class Plotter { throw new IllegalStateException(e); } }); + METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); final Limit limitBy = plotSettings.getLimitBy(); @@ -145,7 +143,7 @@ public class Plotter { } catch (final IOException e) { throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); } finally { - FileUtils.delete(tmpDir); + // XXX TODO revert: FileUtils.delete(tmpDir); LOGGER.trace("done plot"); } } @@ -166,6 +164,7 @@ public class Plotter { final long start = System.nanoTime(); final Stream timeValueStream = groupResult.asStream(); + final Tags groupedBy = groupResult.getGroupedBy(); final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5); @@ -206,21 +205,21 @@ public class Plotter { statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count; // check if value is in the selected y-range - boolean valueIsInYRange = value < minValue || value > maxValue; + final boolean valueIsInYRange = value < minValue || value > maxValue; if (valueIsInYRange) { ignoredValues++; } else { plottedValues++; } - aggregator.addValue(valueIsInYRange, epochMilli, value); + aggregator.addValue(groupedBy, valueIsInYRange, epochMilli, value); } } METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}", plottedValues, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy().asString()); - return new CsvSummary(count, plottedValues, statsMaxValue, statsCurrentAverage, aggregator.getAggregatedData()); + return new CsvSummary(count, plottedValues, statsMaxValue, statsCurrentAverage, aggregator); } @@ -231,22 +230,10 @@ public class Plotter { static String title(final Tags tags, final CsvSummary csvSummary) { - final StringBuilder result = new StringBuilder(); + final StringBuilder result = new StringBuilder(tags.asValueString()); final int values = csvSummary.getValues(); final int plottedValues = csvSummary.getPlottedValues(); - - if (tags.isEmpty()) { - result.append(DEFAULT_GROUP); - } else { - tags.forEach((k, v) -> { - if (result.length() > 0) { - result.append(" / "); - } - result.append(v); - }); - } - result.append(" ("); if (plottedValues != values) { result.append(String.format("%,d / %,d", plottedValues, values)); diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 7d889a7..7f94640 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -3,6 +3,7 @@ package org.lucares.pdbui; import org.lucares.pdb.plot.api.Aggregate; import org.lucares.pdb.plot.api.AggregateHandlerCollection; import org.lucares.pdb.plot.api.AxisScale; +import org.lucares.pdb.plot.api.BarChartHandler; import org.lucares.pdb.plot.api.CumulativeDistributionHandler; import org.lucares.pdb.plot.api.HistogramHandler; import org.lucares.pdb.plot.api.ParallelRequestsAggregate; @@ -64,10 +65,10 @@ class PlotSettingsTransformer { switch (aggregate) { case CUM_DISTRIBUTION: - aggregateHandlerCollection.add(new CumulativeDistributionHandler()); + aggregateHandlerCollection.addAggregateHandler(new CumulativeDistributionHandler()); break; case PARALLEL: - aggregateHandlerCollection.add(new ParallelRequestsAggregate()); + aggregateHandlerCollection.addAggregateHandler(new ParallelRequestsAggregate()); break; case SCATTER: if (yRangeUnit == TimeRangeUnitInternal.AUTOMATIC && yAxisScale == AxisScale.LINEAR) { @@ -76,11 +77,14 @@ class PlotSettingsTransformer { throw new UnsupportedOperationException( "linear axis with automatic y range does not work, use logarthmic y-axis, or define a y-axis range"); } else { - aggregateHandlerCollection.add(new ScatterAggregateHandler()); + aggregateHandlerCollection.addAggregateHandler(new ScatterAggregateHandler()); } break; case HISTOGRAM: - aggregateHandlerCollection.add(new HistogramHandler()); + aggregateHandlerCollection.addAggregateHandler(new HistogramHandler()); + break; + case BAR: + aggregateHandlerCollection.addAggregateHandler(new BarChartHandler()); break; default: throw new IllegalStateException("unhandled enum: " + aggregate); diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/TcpIngestorTest.java b/pdb-ui/src/test/java/org/lucares/pdbui/TcpIngestorTest.java index 897a0a0..c709e66 100644 --- a/pdb-ui/src/test/java/org/lucares/pdbui/TcpIngestorTest.java +++ b/pdb-ui/src/test/java/org/lucares/pdbui/TcpIngestorTest.java @@ -153,7 +153,7 @@ public class TcpIngestorTest { @Test public void testIngestionThreadDoesNotDieOnErrors() throws Exception { - final OffsetDateTime dateA = OffsetDateTime.ofInstant(Instant.ofEpochMilli(-1), ZoneOffset.UTC); + final OffsetDateTime dateA = OffsetDateTime.now().minusMinutes(1); final OffsetDateTime dateB = OffsetDateTime.now(); final DateTimeRange dateRange = new DateTimeRange(dateA, dateB); final String host = "someHost"; @@ -163,7 +163,7 @@ public class TcpIngestorTest { // has a negative epoch time milli and negative value final Map entryA = new HashMap<>(); - entryA.put("duration", -1); + entryA.put("duration", 1); entryA.put("@timestamp", dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME)); entryA.put("host", host); entryA.put("tags", Collections.emptyList()); @@ -194,7 +194,7 @@ public class TcpIngestorTest { Assertions.assertEquals(4, result.size()); Assertions.assertEquals(dateA.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), result.get(0)); - Assertions.assertEquals(-1, result.get(1)); + Assertions.assertEquals(1, result.get(1)); Assertions.assertEquals(dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), result.get(2)); Assertions.assertEquals(2, result.get(3));