andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add bar charts

Browse Source
This commit is contained in:
Andreas Huber
2020-01-19 10:35:07 +01:00
parent 1587046907
commit cf7e5ec968
35 changed files with 539 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java
View File

@@ -33,4 +33,9 @@ public class GroupResult {
return result;
}
@Override
public String toString() {
return groupedBy.toString();
}
}

5
pdb-api/src/main/java/org/lucares/pdb/api/Result.java
View File

@@ -27,4 +27,9 @@ public class Result {
public List<GroupResult> getGroups() {
return new ArrayList<>(groupResults);
}
@Override
public String toString() {
return groupResults.toString();
}
}

25
pdb-api/src/main/java/org/lucares/pdb/api/Tags.java
View File

@@ -14,11 +14,13 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
public class Tags implements Comparable<Tags> {
private static final String DEFAULT_GROUP = "<none>";
public static StringCompressor STRING_COMPRESSOR = null;
private static final byte[] EMPTY_BYTES = new byte[0];
public static final Tags EMPTY = new Tags();
private final List<Tag> tags;
private int hashCode = 0;
public Tags() {
tags = new ArrayList<>();
@@ -216,10 +218,12 @@ public class Tags implements Comparable<Tags> {
@Override
public int hashCode() {
if (hashCode == 0) {
final int prime = 31;
int result = 1;
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
final int result = 1;
hashCode = prime * result + ((tags == null) ? 0 : tags.hashCode());
}
return hashCode;
}
@Override
@@ -278,4 +282,19 @@ public class Tags implements Comparable<Tags> {
return result.toString();
}
public String asValueString() {
final StringBuilder result = new StringBuilder();
if (isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
return result.toString();
}
}

2
pdb-js/src/app/plot.service.ts
View File

@@ -17,6 +17,7 @@ export class PlotService {
this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration));
this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount));
this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count));
this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", true, DataType.Group, DataType.Count));
this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration));
@@ -25,7 +26,6 @@ export class PlotService {
this.plotTypes.push(new PlotType("VIOLIN", "Violin", "violin-chart", false, DataType.Group, DataType.Duration));
this.plotTypes.push(new PlotType("STRIP", "Strip", "strip-chart", false, DataType.Group, DataType.Duration));
this.plotTypes.push(new PlotType("PIE", "Pie", "pie-chart", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("STEP_FIT", "Step Fit", "step-fit", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("LAG", "Lag", "lag-plot", false, DataType.Other, DataType.Other));
this.plotTypes.push(new PlotType("ACF", "ACF", "acf-plot", false, DataType.Other, DataType.Other));

6
pdb-js/src/assets/js/invaders.js
View File

@@ -22,11 +22,13 @@ function initInvaders(parentDivId) {
var parent = $('#'+parentDivId);
var height = parent.height();
var width = parent.width();
$('body').append('<div id="'+invaders_parentDivId+'"><div id="invaders_points">Points: 0</div><div id="invaders_kills">Kills: 0</div><div id="invaders_game_over"><div class="invader_notify">Game Over</div></div></div>');
$('body').append('<canvas id="'+invaders_parentDivId+'"><div id="invaders_points">Points: 0</div><div id="invaders_kills">Kills: 0</div><div id="invaders_game_over"><div class="invader_notify">Game Over</div></div></canvas>');
$('#'+invaders_parentDivId).offset({ top: parent.offset().top, left: parent.offset().left})
$('#'+invaders_parentDivId).height(parent.height());
$('#'+invaders_parentDivId).width(parent.width());
$('#'+invaders_parentDivId)
$('.invader_notify').click(function() {
// restart the game
$('#'+invaders_parentDivId).remove();
@@ -81,7 +83,7 @@ function addInvader()
var top = 10; // start at the top
var left = invaders_margin + random(width-2*invaders_margin); // spread horizontally, but keep margin to left and right
parent.append('<div id="'+id+'" class="invader" style="position:absolute;top:'+top+'px;left:'+left+'px;"/>');
parent.append('<span id="'+id+'" class="invader" style="position:absolute;top:'+top+'px;left:'+left+'px;"/>');
$("#"+id).click(function()
{

5
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java
View File

@@ -5,6 +5,8 @@ package org.lucares.pdb.plot.api;
* drawn.
*/
public enum Aggregate {
BAR,
PARALLEL,
SCATTER,
@@ -16,5 +18,6 @@ public enum Aggregate {
*/
CUM_DISTRIBUTION,
HISTOGRAM
HISTOGRAM,
}

2
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java
View File

@@ -52,7 +52,7 @@ public abstract class AggregateHandler implements Appender {
abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
abstract void addPlot(StringBuilder result, CustomAggregator aggregator, LineStyle lineStyle,
Optional<String> title);
abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,

17
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java
View File

@@ -22,7 +22,7 @@ public class AggregateHandlerCollection {
private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
public void add(final AggregateHandler aggregateHandler) {
public void addAggregateHandler(final AggregateHandler aggregateHandler) {
aggregateHandlers.add(aggregateHandler);
}
@@ -99,14 +99,21 @@ public class AggregateHandlerCollection {
final List<AggregateHandler> handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers,
PLOTTING_ORDER);
for (final AggregateHandler handler : handlersInPlottingOrder) {
long index = 0;
for (final DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty();
final Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData()
.get(handler.getAggregateType());
if (aggregatedData.isPresent()) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title);
final Optional<CustomAggregator> optionalAggregator = dataSerie.getAggregators()
.getCustomAggregator(handler.getAggregateType());
if (optionalAggregator.isPresent()) {
final CustomAggregator aggregator = optionalAggregator.get();
if (aggregator instanceof IndexedAggregator) {
((IndexedAggregator) aggregator).setIndex(index);
index++;
}
handler.addPlot(result, aggregator, dataSerie.getStyle(), title);
}
}

2
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java
View File

@@ -4,6 +4,8 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Optional;
// TODO remove
@Deprecated
public class AggregatedDataCollection implements Iterable<AggregatedData> {
private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>();

27
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java
View File

@@ -1,29 +1,30 @@
package org.lucares.pdb.plot.api;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.lucares.pdb.api.Tags;
public class AggregatorCollection {
private final List<CustomAggregator> aggregators;
private final Map<Aggregate, CustomAggregator> aggregators = new LinkedHashMap<>();
public AggregatorCollection(final List<CustomAggregator> aggregators) {
this.aggregators = aggregators;
for (final CustomAggregator aggregator : aggregators) {
this.aggregators.put(aggregator.getType(), aggregator);
}
}
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
for (final CustomAggregator aggregator : aggregators) {
public void addValue(final Tags groupedBy, final boolean valueIsInYRange, final long epochMilli, final long value) {
for (final CustomAggregator aggregator : aggregators.values()) {
aggregator.addValue(valueIsInYRange, epochMilli, value);
}
}
public AggregatedDataCollection getAggregatedData() throws IOException {
final AggregatedDataCollection result = new AggregatedDataCollection();
for (final CustomAggregator aggregator : aggregators) {
result.put(aggregator.getType(), aggregator.getAggregatedData());
public Optional<CustomAggregator> getCustomAggregator(final Aggregate type) {
return Optional.ofNullable(aggregators.get(type));
}
return result;
}
}

1
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java
View File

@@ -14,4 +14,5 @@ public interface Appender {
default void appendf(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US, format, args));
}
}

69
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartAggregator.java Normal file
View File

@@ -0,0 +1,69 @@
package org.lucares.pdb.plot.api;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.lucares.pdb.api.RuntimeIOException;
public class BarChartAggregator implements CustomAggregator, IndexedAggregator {
long count = 0;
private final Path tmpDir;
private Long index = null;
public BarChartAggregator(final Path tmpDir) {
super();
this.tmpDir = tmpDir;
}
@Override
public void setIndex(final long index) {
this.index = index;
}
@Override
public long getIndex() throws IllegalStateException {
if (this.index == null) {
throw new IllegalStateException("index was not set");
}
return this.index;
}
@Override
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
count++;
}
@Override
public AggregatedData getAggregatedData() {
try {
final File dataFile = File.createTempFile("bar", ".dat", tmpDir.toFile());
final StringBuilder csv = new StringBuilder();
csv.append(index + 0.5);
csv.append(",");
csv.append("");
csv.append(",");
csv.append(count);
csv.append("\n");
Files.writeString(dataFile.toPath(), csv.toString(), StandardCharsets.UTF_8);
final AggregatedData result = new AggregatedData("label", dataFile);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
@Override
public Aggregate getType() {
return Aggregate.BAR;
}
}

97
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartHandler.java Normal file
View File

@@ -0,0 +1,97 @@
package org.lucares.pdb.plot.api;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotLineType;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
public class BarChartHandler extends AggregateHandler {
@Override
Type getAxisType(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
return Type.Group;
case Y1:
case Y2:
return Type.Number;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
@Override
Aggregate getAggregateType() {
return Aggregate.BAR;
}
@Override
AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Bar");
result.setType(Type.Group);
result.setAxis(getxAxis());
result.setTicsEnabled(true);
result.setFrom("0");
result.setTo(String.valueOf(dataSeries.size()));
final List<String> ticsLabels = new ArrayList<>();
int index = 1;
for (final DataSeries series : dataSeries) {
ticsLabels.add("\"" + series.getTitle() + "\" " + index);
index++;
}
// result.setTics(ticsLabels);
return result;
}
@Override
AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Count");
result.setType(Type.Number);
result.setAxis(getyAxis());
result.setTicsEnabled(true);
result.setFrom("0");
result.setLogscale(settings.getYAxisScale() == AxisScale.LOG10);
return result;
}
@Override
void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle,
final Optional<String> title) {
final AggregatedData aggregatedData = aggregator.getAggregatedData();
/*
* appendfln(result,
* "'%s' using 1:3:xtic(2) notitle with %s axes %s fs solid %s, \\", //
* aggregatedData.getDataFile(), // GnuplotLineType.Bar, // gnuplotXYAxis(), //
* lineStyle// );
*/
appendfln(result, "'%s' using 1:3:xtic(2) %s with %s axes %s fs solid %s, \\", //
aggregatedData.getDataFile(), //
gnuplotTitle(title), //
GnuplotLineType.Bar, //
gnuplotXYAxis(), //
lineStyle.brighter()//
);
}
@Override
CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new BarChartAggregator(tmpDir);
}
}

9
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java
View File

@@ -12,6 +12,7 @@ import java.util.LinkedHashMap;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
import org.lucares.pdb.api.RuntimeIOException;
public class CumulativeDistributionCustomAggregator implements CustomAggregator {
@@ -78,13 +79,14 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
}
@Override
public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) {
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
map.compute(value, 0, l -> l + 1);
totalValues++;
}
@Override
public AggregatedData getAggregatedData() throws IOException {
public AggregatedData getAggregatedData() {
try {
final char separator = ',';
final char newline = '\n';
@@ -118,6 +120,9 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
final String title = String.format("cumulative distribution");
return new AggregatedData(title, dataFile);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
@Override

27
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java
View File

@@ -4,18 +4,18 @@ import java.nio.file.Path;
import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
import org.lucares.recommind.logs.AxisTime;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
public class CumulativeDistributionHandler extends AggregateHandler {
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir);
}
@@ -24,7 +24,7 @@ public class CumulativeDistributionHandler extends AggregateHandler {
}
@Override
Type getAxisType(GnuplotAxis axis) {
Type getAxisType(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
@@ -38,15 +38,15 @@ public class CumulativeDistributionHandler extends AggregateHandler {
}
@Override
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = new AxisSettings();
public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Cumulative Distribution");
result.setType(Type.Percent);
result.setAxis(getxAxis());
@@ -57,8 +57,8 @@ public class CumulativeDistributionHandler extends AggregateHandler {
return result;
}
private int computeTicIncrement(GnuplotSettings settings) {
int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
private int computeTicIncrement(final GnuplotSettings settings) {
final int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
if (widthByFontSize < 50) {
return 20;
} else if (widthByFontSize < 75) {
@@ -69,8 +69,11 @@ public class CumulativeDistributionHandler extends AggregateHandler {
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle,
final Optional<String> title) {
final AggregatedData aggregatedData = aggregator.getAggregatedData();
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //

4
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomAggregator.java
View File

@@ -1,12 +1,10 @@
package org.lucares.pdb.plot.api;
import java.io.IOException;
public interface CustomAggregator {
void addValue(boolean valueIsInYRange, long epochMilli, long value);
AggregatedData getAggregatedData() throws IOException;
AggregatedData getAggregatedData();
Aggregate getType();
}

14
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CustomGroupedAggregator.java Normal file
View File

@@ -0,0 +1,14 @@
//package org.lucares.pdb.plot.api;
//
//import java.io.IOException;
//import java.util.Optional;
//
//import org.lucares.pdb.api.Tags;
//
//public interface CustomGroupedAggregator {
// void addValue(Tags groupedBy, boolean valueIsInYRange, long epochMilli, long value);
//
// Optional<AggregatedData> getAggregatedData() throws IOException;
//
// Aggregate getType();
//}

7
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java
View File

@@ -11,6 +11,7 @@ import java.nio.file.Path;
import org.lucares.collections.LongLongConsumer;
import org.lucares.collections.LongLongHashMap;
import org.lucares.pdb.api.RuntimeIOException;
public class HistogramAggregator implements CustomAggregator {
@@ -57,7 +58,8 @@ public class HistogramAggregator implements CustomAggregator {
}
@Override
public AggregatedData getAggregatedData() throws IOException {
public AggregatedData getAggregatedData() {
try {
final char separator = ',';
final char newline = '\n';
@@ -91,6 +93,9 @@ public class HistogramAggregator implements CustomAggregator {
}
final AggregatedData result = new AggregatedData("histogram", dataFile);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
@Override

4
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java
View File

@@ -54,8 +54,10 @@ public class HistogramHandler extends AggregateHandler {
}
@Override
void addPlot(final StringBuilder result, final AggregatedData aggregatedData, final LineStyle lineStyle,
void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle,
final Optional<String> title) {
final AggregatedData aggregatedData = aggregator.getAggregatedData();
appendfln(result, "'%s' using 1:2 %s with boxes axes %s lw 1 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //

16
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/IndexedAggregator.java Normal file
View File

@@ -0,0 +1,16 @@
package org.lucares.pdb.plot.api;
public interface IndexedAggregator {
/**
* Set the index of this {@link CustomAggregator}.
*/
public void setIndex(long index);
/**
* Returns the index.
*
* @throws IllegalStateException if the index was no set
*/
public long getIndex() throws IllegalStateException;
}

66
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ObsoleteBarChartAggregator.java Normal file
View File

@@ -0,0 +1,66 @@
//package org.lucares.pdb.plot.api;
//
//import java.io.File;
//import java.io.IOException;
//import java.nio.charset.StandardCharsets;
//import java.nio.file.Files;
//import java.nio.file.Path;
//import java.util.Map;
//import java.util.Map.Entry;
//import java.util.Optional;
//import java.util.concurrent.ConcurrentHashMap;
//import java.util.concurrent.atomic.AtomicLong;
//
//import org.lucares.pdb.api.Tags;
//
//public class ObsoleteBarChartAggregator implements CustomGroupedAggregator {
//
// Map<Tags, AtomicLong> counts = new ConcurrentHashMap<>();
//
// private final Path tmpDir;
//
// public ObsoleteBarChartAggregator(final Path tmpDir) {
// super();
// this.tmpDir = tmpDir;
// }
//
// @Override
// public void addValue(final Tags groupedBy, final boolean valueIsInYRange, final long epochMilli, final long value) {
// final AtomicLong val = counts.computeIfAbsent(groupedBy, (k) -> new AtomicLong());
// val.incrementAndGet();
// }
//
// @Override
// public Optional<AggregatedData> getAggregatedData() throws IOException {
//
// if (counts.entrySet().isEmpty()) {
// return Optional.empty();
// }
//
// final File dataFile = File.createTempFile("bar", ".dat", tmpDir.toFile());
//
// final StringBuilder csv = new StringBuilder();
//
// int position = 1;
// for (final Entry<Tags, AtomicLong> e : counts.entrySet()) {
// csv.append(position);
// csv.append(",");
// csv.append(e.getKey().asValueString());
// csv.append(",");
// csv.append(e.getValue().get());
// csv.append("\n");
//
// position++;
// }
//
// Files.writeString(dataFile.toPath(), csv.toString(), StandardCharsets.UTF_8);
// final AggregatedData result = new AggregatedData("label", dataFile);
// return Optional.of(result);
// }
//
// @Override
// public Aggregate getType() {
// return Aggregate.BAR;
// }
//
//}

19
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregate.java
View File

@@ -5,18 +5,18 @@ import java.util.Collection;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
import org.lucares.recommind.logs.AxisTime;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
public class ParallelRequestsAggregate extends AggregateHandler {
@Override
Type getAxisType(GnuplotAxis axis) {
Type getAxisType(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
@@ -30,7 +30,7 @@ public class ParallelRequestsAggregate extends AggregateHandler {
}
@Override
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = new AxisSettings();
result.setLabel("Parallel Requests");
result.setType(Type.Number);
@@ -41,15 +41,16 @@ public class ParallelRequestsAggregate extends AggregateHandler {
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = AxisTime.createXAxis(settings);
result.setAxis(getxAxis());
return result;
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle,
final Optional<String> title) {
final AggregatedData aggregatedData = aggregator.getAggregatedData();
appendfln(result, "'%s' using 1:2 %s with filledcurve axes %s lw 1 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
@@ -59,7 +60,7 @@ public class ParallelRequestsAggregate extends AggregateHandler {
}
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
if ((toEpochMilli - fromEpochMilli) <= TimeUnit.HOURS.toMillis(50)) {
return new ParallelRequestsAggregator(tmpDir, fromEpochMilli, toEpochMilli);

14
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ParallelRequestsAggregator.java
View File

@@ -10,6 +10,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Locale;
import org.lucares.pdb.api.RuntimeIOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +41,7 @@ public class ParallelRequestsAggregator implements CustomAggregator {
}
@Override
public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) {
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
final int endPos = (int) (epochMilli - fromEpochMilli);
increments[endPos]--;
@@ -51,8 +52,8 @@ public class ParallelRequestsAggregator implements CustomAggregator {
}
@Override
public AggregatedData getAggregatedData() throws IOException {
public AggregatedData getAggregatedData() {
try {
final long start = System.nanoTime();
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
@@ -82,9 +83,12 @@ public class ParallelRequestsAggregator implements CustomAggregator {
}
final String title = String.format("parallelRequests");
METRICS_LOGGER.debug("wrote parallelRequests csv in: {}ms file={}", (System.nanoTime() - start) / 1_000_000.0,
dataFile);
METRICS_LOGGER.debug("wrote parallelRequests csv in: {}ms file={}",
(System.nanoTime() - start) / 1_000_000.0, dataFile);
return new AggregatedData(title, dataFile);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private void appendTimeAndValue(final StringBuilder builder, final long timeEpochMilli, final int value) {

17
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregateHandler.java
View File

@@ -5,18 +5,18 @@ import java.util.Collection;
import java.util.Optional;
import org.lucares.recommind.logs.AxisSettings;
import org.lucares.recommind.logs.AxisSettings.Type;
import org.lucares.recommind.logs.AxisTime;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.GnuplotAxis;
import org.lucares.recommind.logs.GnuplotLineType;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LineStyle;
import org.lucares.recommind.logs.AxisSettings.Type;
public class ScatterAggregateHandler extends AggregateHandler {
@Override
Type getAxisType(GnuplotAxis axis) {
Type getAxisType(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
@@ -30,23 +30,24 @@ public class ScatterAggregateHandler extends AggregateHandler {
}
@Override
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
public AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
public AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
final AxisSettings result = AxisTime.createXAxis(settings);
result.setAxis(getxAxis());
return result;
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
public void addPlot(final StringBuilder result, final CustomAggregator aggregator, final LineStyle lineStyle,
final Optional<String> title) {
final AggregatedData aggregatedData = aggregator.getAggregatedData();
appendfln(result, "'%s' using 1:2 %s with %s axes %s %s, \\", //
aggregatedData.getDataFile(), //
gnuplotTitle(title), //
@@ -57,8 +58,8 @@ public class ScatterAggregateHandler extends AggregateHandler {
}
@Override
public CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli) {
public CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new ScatterAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli);
}

17
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/ScatterAggregator.java
View File

@@ -12,6 +12,7 @@ import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.lucares.collections.Sparse2DLongArray;
import org.lucares.pdb.api.RuntimeIOException;
import org.lucares.recommind.logs.GnuplotSettings;
import org.lucares.recommind.logs.LambdaFriendlyWriter;
import org.lucares.recommind.logs.LongUtils;
@@ -29,9 +30,10 @@ public class ScatterAggregator implements CustomAggregator {
private final long maxValue;
private final long durationMillisPerPixel;
private Path tmpDir;
private final Path tmpDir;
public ScatterAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, long toEpochMilli) {
public ScatterAggregator(final Path tmpDir, final PlotSettings plotSettings, final long fromEpochMilli,
final long toEpochMilli) {
this.tmpDir = tmpDir;
useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
@@ -49,20 +51,20 @@ public class ScatterAggregator implements CustomAggregator {
}
@Override
public void addValue(boolean valueIsInYRange, long epochMilli, long value) {
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
final long roundedEpochMilli = epochMilli - epochMilli % epochMillisPerPixel;
final long roundedValue = value - value % durationMillisPerPixel;
matrix2d.put(roundedEpochMilli, roundedValue, 1);
}
@Override
public AggregatedData getAggregatedData() throws IOException {
public AggregatedData getAggregatedData() {
try {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
final int separator = ',';
final int newline = '\n';
long[] actualValuesWritten = new long[1];
final long[] actualValuesWritten = new long[1];
final StringBuilder formattedDateBuilder = new StringBuilder();
try (final LambdaFriendlyWriter output = new LambdaFriendlyWriter(new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.ISO_8859_1)));
@@ -90,6 +92,9 @@ public class ScatterAggregator implements CustomAggregator {
}
return new AggregatedData("scatter", dataFile);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
@Override

2
pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java
View File

@@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
public class AxisSettings {
public enum Type {
Number, Time, Duration, Percent, HistogramBin, HistogramCount
Number, Time, Duration, Percent, HistogramBin, HistogramCount, Group
}
private String format = "";

12
pdb-plotting/src/main/java/org/lucares/recommind/logs/CsvSummary.java
View File

@@ -1,22 +1,22 @@
package org.lucares.recommind.logs;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
import org.lucares.pdb.plot.api.AggregatorCollection;
class CsvSummary {
private final int values;
private final long maxValue;
private final AggregatedDataCollection aggregatedData;
private final AggregatorCollection aggregators;
private final double statsAverage;
private final int plottedValues;
public CsvSummary(final int values, final int plottedValues, final long maxValue, final double statsAverage,
final AggregatedDataCollection aggregatedData) {
final AggregatorCollection aggregators) {
super();
this.values = values;
this.plottedValues = plottedValues;
this.maxValue = maxValue;
this.statsAverage = statsAverage;
this.aggregatedData = aggregatedData;
this.aggregators = aggregators;
}
/**
@@ -47,7 +47,7 @@ class CsvSummary {
return statsAverage;
}
public AggregatedDataCollection getAggregatedData() {
return aggregatedData;
public AggregatorCollection getAggregators() {
return aggregators;
}
}

4
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -6,7 +6,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
import org.lucares.pdb.plot.api.AggregatorCollection;
import org.lucares.pdb.plot.api.Limit;
public interface DataSeries {
@@ -41,7 +41,7 @@ public interface DataSeries {
public LineStyle getStyle();
public AggregatedDataCollection getAggregatedData();
public AggregatorCollection getAggregators();
public static Map<String, Integer> toMap(final List<DataSeries> dataSeries) {
final Map<String, Integer> result = new LinkedHashMap<>();

6
pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java
View File

@@ -1,6 +1,6 @@
package org.lucares.recommind.logs;
import org.lucares.pdb.plot.api.AggregatedDataCollection;
import org.lucares.pdb.plot.api.AggregatorCollection;
public class FileBackedDataSeries implements DataSeries {
@@ -64,7 +64,7 @@ public class FileBackedDataSeries implements DataSeries {
}
@Override
public AggregatedDataCollection getAggregatedData() {
return csvSummary.getAggregatedData();
public AggregatorCollection getAggregators() {
return csvSummary.getAggregators();
}
}

6
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotColorPalettes.java
View File

@@ -37,9 +37,9 @@ public interface GnuplotColorPalettes {
);
List<GnuplotColor> GNUPLOT_REORDERED = Arrays.asList(//
GnuplotColor.byHex("0072b2"), // blue
GnuplotColor.byHex("e69f00"), // orange
GnuplotColor.byHex("9400D3"), // purple
GnuplotColor.byHex("0072b2"), // blue 00A2FF, 0091E6, 0072B2, 005180, 002840
GnuplotColor.byHex("e69f00"), // orange FFAE00, E69F00, BF8300, 805700, 402C00
GnuplotColor.byHex("9400D3"), // purple B300FF, A100E6, 9400D3, 590080, 2D0040
GnuplotColor.byHex("009e73"), // green
GnuplotColor.byHex("f0e442"), // yellow
GnuplotColor.byHex("e51e10"), // red

3
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -64,6 +64,9 @@ public class GnuplotFileGenerator implements Appender {
}
}
// appendfln(result, "set xrange [-1:1]");
appendfln(result, "set boxwidth 0.5");
appendf(result, "plot ");
settings.getAggregates().addPlots(result, dataSeries);

4
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java
View File

@@ -3,11 +3,13 @@ package org.lucares.recommind.logs;
public enum GnuplotLineType {
LINE("line"),
Bar("boxes"),
Points("points");
private String gnuplotLineType;
GnuplotLineType(String gnuplotLineType) {
GnuplotLineType(final String gnuplotLineType) {
this.gnuplotLineType = gnuplotLineType;
}

27
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -26,7 +26,6 @@ import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.TimeRangeUnitInternal;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,8 +34,6 @@ public class Plotter {
private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter");
static final String DEFAULT_GROUP = "<none>";
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
@@ -95,6 +92,7 @@ public class Plotter {
throw new IllegalStateException(e);
}
});
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
final Limit limitBy = plotSettings.getLimitBy();
@@ -145,7 +143,7 @@ public class Plotter {
} catch (final IOException e) {
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
} finally {
FileUtils.delete(tmpDir);
// XXX TODO revert: FileUtils.delete(tmpDir);
LOGGER.trace("done plot");
}
}
@@ -166,6 +164,7 @@ public class Plotter {
final long start = System.nanoTime();
final Stream<LongList> timeValueStream = groupResult.asStream();
final Tags groupedBy = groupResult.getGroupedBy();
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
@@ -206,21 +205,21 @@ public class Plotter {
statsCurrentAverage = statsCurrentAverage + (value - statsCurrentAverage) / count;
// check if value is in the selected y-range
boolean valueIsInYRange = value < minValue || value > maxValue;
final boolean valueIsInYRange = value < minValue || value > maxValue;
if (valueIsInYRange) {
ignoredValues++;
} else {
plottedValues++;
}
aggregator.addValue(valueIsInYRange, epochMilli, value);
aggregator.addValue(groupedBy, valueIsInYRange, epochMilli, value);
}
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}",
plottedValues, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis),
groupResult.getGroupedBy().asString());
return new CsvSummary(count, plottedValues, statsMaxValue, statsCurrentAverage, aggregator.getAggregatedData());
return new CsvSummary(count, plottedValues, statsMaxValue, statsCurrentAverage, aggregator);
}
@@ -231,22 +230,10 @@ public class Plotter {
static String title(final Tags tags, final CsvSummary csvSummary) {
final StringBuilder result = new StringBuilder();
final StringBuilder result = new StringBuilder(tags.asValueString());
final int values = csvSummary.getValues();
final int plottedValues = csvSummary.getPlottedValues();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append(" (");
if (plottedValues != values) {
result.append(String.format("%,d / %,d", plottedValues, values));

12
pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java
View File

@@ -3,6 +3,7 @@ package org.lucares.pdbui;
import org.lucares.pdb.plot.api.Aggregate;
import org.lucares.pdb.plot.api.AggregateHandlerCollection;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.BarChartHandler;
import org.lucares.pdb.plot.api.CumulativeDistributionHandler;
import org.lucares.pdb.plot.api.HistogramHandler;
import org.lucares.pdb.plot.api.ParallelRequestsAggregate;
@@ -64,10 +65,10 @@ class PlotSettingsTransformer {
switch (aggregate) {
case CUM_DISTRIBUTION:
aggregateHandlerCollection.add(new CumulativeDistributionHandler());
aggregateHandlerCollection.addAggregateHandler(new CumulativeDistributionHandler());
break;
case PARALLEL:
aggregateHandlerCollection.add(new ParallelRequestsAggregate());
aggregateHandlerCollection.addAggregateHandler(new ParallelRequestsAggregate());
break;
case SCATTER:
if (yRangeUnit == TimeRangeUnitInternal.AUTOMATIC && yAxisScale == AxisScale.LINEAR) {
@@ -76,11 +77,14 @@ class PlotSettingsTransformer {
throw new UnsupportedOperationException(
"linear axis with automatic y range does not work, use logarthmic y-axis, or define a y-axis range");
} else {
aggregateHandlerCollection.add(new ScatterAggregateHandler());
aggregateHandlerCollection.addAggregateHandler(new ScatterAggregateHandler());
}
break;
case HISTOGRAM:
aggregateHandlerCollection.add(new HistogramHandler());
aggregateHandlerCollection.addAggregateHandler(new HistogramHandler());
break;
case BAR:
aggregateHandlerCollection.addAggregateHandler(new BarChartHandler());
break;
default:
throw new IllegalStateException("unhandled enum: " + aggregate);

6
pdb-ui/src/test/java/org/lucares/pdbui/TcpIngestorTest.java
View File

@@ -153,7 +153,7 @@ public class TcpIngestorTest {
@Test
public void testIngestionThreadDoesNotDieOnErrors() throws Exception {
final OffsetDateTime dateA = OffsetDateTime.ofInstant(Instant.ofEpochMilli(-1), ZoneOffset.UTC);
final OffsetDateTime dateA = OffsetDateTime.now().minusMinutes(1);
final OffsetDateTime dateB = OffsetDateTime.now();
final DateTimeRange dateRange = new DateTimeRange(dateA, dateB);
final String host = "someHost";
@@ -163,7 +163,7 @@ public class TcpIngestorTest {
// has a negative epoch time milli and negative value
final Map<String, Object> entryA = new HashMap<>();
entryA.put("duration", -1);
entryA.put("duration", 1);
entryA.put("@timestamp", dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
entryA.put("host", host);
entryA.put("tags", Collections.emptyList());
@@ -194,7 +194,7 @@ public class TcpIngestorTest {
Assertions.assertEquals(4, result.size());
Assertions.assertEquals(dateA.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), result.get(0));
Assertions.assertEquals(-1, result.get(1));
Assertions.assertEquals(1, result.get(1));
Assertions.assertEquals(dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), result.get(2));
Assertions.assertEquals(2, result.get(3));