andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add links to current settings and to current image

Browse Source
This commit is contained in:
Andreas Huber
2018-04-10 19:59:09 +02:00
parent fe29b0d738
commit 7018a11ab3
7 changed files with 198 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java
View File

@@ -29,24 +29,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PercentilePlot implements ConcretePlotter {
private static final Logger LOGGER = LoggerFactory
.getLogger(ScatterPlot.class);
private static final Logger METRICS_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.plotter.percentile");
private PerformanceDb db;
private Path tmpBaseDir;
private Path outputDir;
private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.percentile");
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
public PercentilePlot(PerformanceDb db, final Path tmpBaseDir,
Path outputDir) {
public PercentilePlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
this.db = db;
this.tmpBaseDir = tmpBaseDir;
this.outputDir = outputDir;
}
@Override
public PlotResult plot(PlotSettings plotSettings)
throws InternalPlottingException {
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
LOGGER.trace("start plot: {}", plotSettings);
@@ -54,8 +50,7 @@ public class PercentilePlot implements ConcretePlotter {
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
try {
Files.createDirectories(tmpDir);
final List<DataSeries> dataSeries = Collections
.synchronizedList(new ArrayList<>());
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
final String query = plotSettings.getQuery();
final List<String> groupBy = plotSettings.getGroupBy();
@@ -68,43 +63,35 @@ public class PercentilePlot implements ConcretePlotter {
final long start = System.nanoTime();
final AtomicInteger idCounter = new AtomicInteger(0);
result.getGroups()
.stream()
.parallel()
.forEach(
groupResult -> {
try {
final int id = idCounter.getAndIncrement();
result.getGroups().stream().parallel().forEach(groupResult -> {
try {
final int id = idCounter.getAndIncrement();
final FileBackedDataSeries dataSerie = toCsv(
id, groupResult, tmpDir, dateFrom,
dateTo, plotSettings);
final FileBackedDataSeries dataSerie = toCsv(id, groupResult, tmpDir, dateFrom, dateTo,
plotSettings);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
} catch (Exception e) {
throw new IllegalStateException(e); // TODO
// handle
}
});
METRICS_LOGGER.debug("csv generation took: "
+ (System.nanoTime() - start) / 1_000_000.0 + "ms");
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
} catch (final Exception e) {
throw new IllegalStateException(e); // TODO
// handle
}
});
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
if (dataSeries.isEmpty()) {
throw new NoDataPointsException();
}
final Limit limitBy = plotSettings.getLimitBy();
int limit = plotSettings.getLimit();
final int limit = plotSettings.getLimit();
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
DataSeries.setColors(dataSeries);
final Path outputFile = Files.createTempFile(outputDir, "out",
".png");
final Path outputFile = Files.createTempFile(outputDir, "out", ".png");
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
final GnuplotSettings gnuplotSettings = new GnuplotSettings(
outputFile);
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
gnuplotSettings.setHeight(height);
gnuplotSettings.setWidth(width);
defineXAxis(gnuplotSettings);
@@ -114,22 +101,21 @@ public class PercentilePlot implements ConcretePlotter {
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
gnuplot.plot(gnuplotSettings, dataSeries);
return new PlotResult(outputFile.getFileName(), dataSeries);
return new PlotResult(outputFile, dataSeries);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Plotting was interrupted.");
} catch (final IOException e) {
throw new InternalPlottingException("Plotting failed: "
+ e.getMessage(), e);
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
} finally {
FileUtils.delete(tmpDir);
LOGGER.trace("done plot");
}
}
private FileBackedDataSeries toCsv(int id, GroupResult groupResult,
Path tmpDir, OffsetDateTime dateFrom, OffsetDateTime dateTo,
PlotSettings plotSettings) throws IOException {
private FileBackedDataSeries toCsv(final int id, final GroupResult groupResult, final Path tmpDir,
final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings)
throws IOException {
final long start = System.nanoTime();
final Stream<Entry> entries = groupResult.asStream();
@@ -143,13 +129,11 @@ public class PercentilePlot implements ConcretePlotter {
final IntList values = new IntList(); // TODO should be a LongList
long maxValue = 0;
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
final Entry entry = it.next();
long epochMilli = entry.getEpochMilli();
final long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
final long value = entry.getValue();
@@ -160,20 +144,19 @@ public class PercentilePlot implements ConcretePlotter {
}
}
values.parallelSort();
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (values.size() > 0) {
// compute the percentiles
for (int i = 0; i < 100; i++) {
data.append(i);
data.append(separator);
data.append(values.get((int) Math.floor(values.size()
/ 100.0 * i)));
data.append(values.get((int) Math.floor(values.size() / 100.0 * i)));
data.append(newline);
}
maxValue = values.get(values.size() - 1);
@@ -184,19 +167,16 @@ public class PercentilePlot implements ConcretePlotter {
}
output.write(data.toString());
}
METRICS_LOGGER
.debug("wrote {} values to csv in: {}ms (ignored {} values) grouping={}",
count, (System.nanoTime() - start) / 1_000_000.0,
ignoredValues, groupResult.getGroupedBy());
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) grouping={}", count,
(System.nanoTime() - start) / 1_000_000.0, ignoredValues, groupResult.getGroupedBy());
final String title = ConcretePlotter.title(groupResult.getGroupedBy(),
values.size());
final String title = ConcretePlotter.title(groupResult.getGroupedBy(), values.size());
CsvSummary csvSummary = new CsvSummary(dataFile, values.size(), maxValue, null);
return new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.LINE);
final CsvSummary csvSummary = new CsvSummary(dataFile, values.size(), maxValue, null);
return new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.LINE);
}
private void defineXAxis(GnuplotSettings gnuplotSettings) {
private void defineXAxis(final GnuplotSettings gnuplotSettings) {
final XAxisSettings xAxis = gnuplotSettings.getxAxisSettings();
xAxis.setxDataTime(false);
xAxis.setFrom("0");

12
pdb-plotting/src/main/java/org/lucares/recommind/logs/PlotResult.java
View File

@@ -4,17 +4,21 @@ import java.nio.file.Path;
import java.util.List;
public class PlotResult {
private final Path imageName;
private final Path imagePath;
private final List<DataSeries> dataSeries;
public PlotResult(final Path imageName, final List<DataSeries> dataSeries) {
public PlotResult(final Path imagePath, final List<DataSeries> dataSeries) {
super();
this.imageName = imageName;
this.imagePath = imagePath;
this.dataSeries = dataSeries;
}
public Path getImageName() {
return imageName;
return imagePath.getFileName();
}
public Path getImagePath() {
return imagePath;
}
public List<DataSeries> getDataSeries() {

2
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java
View File

@@ -118,7 +118,7 @@ public class ScatterPlot implements ConcretePlotter {
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
gnuplot.plot(gnuplotSettings, dataSeries);
return new PlotResult(outputFile.getFileName(), dataSeries);
return new PlotResult(outputFile, dataSeries);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Plotting was interrupted.");

18
pdb-ui/src/main/java/org/lucares/pdbui/InternalServerError.java Normal file
View File

@@ -0,0 +1,18 @@
package org.lucares.pdbui;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR, reason = "Internal Server Error")
public class InternalServerError extends RuntimeException {
private static final long serialVersionUID = 548651821080252932L;
public InternalServerError(final String message, final Throwable cause) {
super(message, cause);
}
public InternalServerError(final Throwable cause) {
super(cause);
}
}

65
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -1,5 +1,7 @@
package org.lucares.pdbui;
import java.io.FileInputStream;
import java.io.OutputStream;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collections;
@@ -12,7 +14,11 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.StringUtils;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.PlotType;
import org.lucares.pdbui.domain.Aggregate;
import org.lucares.pdbui.domain.AutocompleteProposal;
import org.lucares.pdbui.domain.AutocompleteProposalByValue;
import org.lucares.pdbui.domain.AutocompleteResponse;
@@ -31,6 +37,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.util.StreamUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
@@ -39,6 +46,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
@Controller
@EnableAutoConfiguration
@@ -105,6 +113,63 @@ public class PdbController implements HardcodedValues {
}
}
@RequestMapping(path = "/plots", //
method = RequestMethod.GET, //
produces = MediaType.APPLICATION_OCTET_STREAM_VALUE //
)
StreamingResponseBody createPlotImage(@RequestParam(name = "query", defaultValue = "") final String query,
@RequestParam(name = "groupBy[]", defaultValue = "") final List<String> aGroupBy,
@RequestParam(name = "limitBy.number", defaultValue = "10") final int limit,
@RequestParam(name = "limitBy.selected", defaultValue = "NO_LIMIT") final Limit limitBy,
@RequestParam(name = "dateFrom", defaultValue = "") final String dateFrom,
@RequestParam(name = "dateRange", defaultValue = "1 week") final String dateRange,
@RequestParam(name = "axisScale", defaultValue = "LINEAR") final AxisScale axisScale,
@RequestParam(name = "plotType", defaultValue = "SCATTER") final PlotType plotType,
@RequestParam(name = "aggregate", defaultValue = "NONE") final Aggregate aggregate,
@RequestParam(name = "keyOutside", defaultValue = "false") final boolean keyOutside,
@RequestParam(name = "height", defaultValue = "1080") final int height,
@RequestParam(name = "width", defaultValue = "1920") final int hidth) {
return (final OutputStream outputStream) -> {
if (StringUtils.isBlank(query)) {
throw new BadRequest("The query must not be empty!");
}
final PlotSettings plotSettings = new PlotSettings();
plotSettings.setQuery(query);
plotSettings.setGroupBy(aGroupBy);
plotSettings.setHeight(height);
plotSettings.setWidth(hidth);
plotSettings.setLimit(limit);
plotSettings.setLimitBy(limitBy);
plotSettings.setDateFrom(dateFrom);
plotSettings.setDateRange(dateRange);
plotSettings.setYAxisScale(axisScale);
plotSettings.setPlotType(plotType);
plotSettings.setAggregate(PlotSettingsTransformer.toAggregateInternal(aggregate));
plotSettings.setKeyOutside(keyOutside);
if (plotterLock.tryLock()) {
try {
final PlotResult result = plotter.plot(plotSettings);
try (FileInputStream in = new FileInputStream(result.getImagePath().toFile())) {
StreamUtils.copy(in, outputStream);
}
} catch (final NoDataPointsException e) {
throw new NotFoundException(e);
} catch (final InternalPlottingException e) {
throw new InternalServerError(e);
} finally {
plotterLock.unlock();
}
} else {
throw new ServiceUnavailableException("Too many parallel requests!");
}
};
}
@RequestMapping(path = "/autocomplete", //
method = RequestMethod.GET, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //

10
pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java
View File

@@ -23,15 +23,17 @@ class PlotSettingsTransformer {
result.setYAxisScale(request.getAxisScale());
result.setPlotType(request.getPlotType());
result.setAggregate(toAggregateInternal(request.getAggregate()));
result.setKeyOutside(request.isKeyOutside());
result.setKeyOutside(request.isKeyOutside());
return result;
}
private static AggregateHandler toAggregateInternal(Aggregate aggregate) {
static AggregateHandler toAggregateInternal(final Aggregate aggregate) {
switch (aggregate) {
case NONE:return new NullAggregate();
case PERCENTILES:return new PercentileAggregate();
case NONE:
return new NullAggregate();
case PERCENTILES:
return new PercentileAggregate();
}
throw new IllegalStateException("unhandled enum: " + aggregate);
}

82
pdb-ui/src/main/resources/resources/js/ui.js
View File

@@ -200,7 +200,7 @@ Vue.component('result-view', {
},
computed: {
showPrevNext: function() {
return data.searchBar.splitBy.values.length > 0 && data.resultView.imageUrl;
return data.searchBar.splitBy.values.length > 0 && !data.resultView.loadingGameActive;
}
},
template: `
@@ -445,13 +445,12 @@ Vue.component('search-bar', {
},
computed: {
permalink: function() {
var groupBy = [];
data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}});
var params = {
'query': data.searchBar.query,
'groupBy': groupBy,
var groupBy = [];
data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}});
var params = {
'query': data.searchBar.query,
'groupBy': groupBy,
'splitByKeys.selected': data.searchBar.splitByKeys.selected,
'limitBy.selected': data.searchBar.limitBy.selected,
'limitBy.number': data.searchBar.limitBy.number,
@@ -459,12 +458,12 @@ Vue.component('search-bar', {
'dateRange': data.searchBar.dateRange,
'axisScale': data.searchBar.axisScale,
'plotType': data.searchBar.plotType,
'showAggregate': data.searchBar.showAggregate,
'aggregate': data.searchBar.aggregate,
'keyOutside': data.searchBar.keyOutside,
};
var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params );
return link;
};
var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params );
return link;
}
},
template: `
@@ -554,7 +553,7 @@ Vue.component('search-bar', {
<div class="group" id="group-show-aggregate" v-show="searchBar.plotType == 'SCATTER'">
<label for="show-aggregate">Aggregate:</label>
<select id="show-aggregate" v-model="searchBar.showAggregate">
<select id="show-aggregate" v-model="searchBar.aggregate">
<option value="NONE">-</option>
<option value="PERCENTILES">percentiles</option>
</select>
@@ -578,7 +577,9 @@ Vue.component('search-bar', {
v-on:click.prevent.stop="dashboard"
><i class="fa fa-object-group" aria-hidden="true"></i> Dashboard</button>
-->
<a v-bind:href="permalink" title="Permanent Link" class="permalink"><i class="fa fa-link" aria-hidden="true"></i></a>
<a v-bind:href="permalink" v-show="permalink" title="Permanent link to the current settings." class="permalink"><i class="fa fa-sliders" aria-hidden="true"></i></a>
<a v-bind:href="searchBar.imagelink" v-show="searchBar.imagelink" title="Image Link" class="permalink"><i class="fa fa-image" aria-hidden="true"></i></a>
</div>
</div>
</form>`
@@ -638,7 +639,7 @@ var data = {
dateRange: GetURLParameter('dateRange','1 week'),
axisScale: GetURLParameter('axisScale','LOG10'),
plotType: GetURLParameter('plotType','SCATTER'),
showAggregate: GetURLParameter('showAggregate','NONE'),
aggregate: GetURLParameter('aggregate','NONE'),
keyOutside: GetURLParameterBoolean('keyOutside', 'false'),
splitBy: {
@@ -646,12 +647,14 @@ var data = {
query: '',
values: [],
index: 0
}
},
imagelink: ""
},
resultView: {
imageUrl: '',
errorMessage: ''
}
errorMessage: '',
loadingGameActive: false
},
};
@@ -659,10 +662,17 @@ function showLoadingIcon()
{
data.resultView.imageUrl = '';
data.resultView.errorMessage = '';
data.resultView.loadingGameActive = true;
startInvaders();
}
function hideLoadingIcon()
{
data.resultView.loadingGameActive = false;
pauseInvaders();
}
function plotCurrent()
{
showLoadingIcon();
@@ -712,14 +722,15 @@ function sendPlotRequest(query){
request['dateRange'] = data.searchBar.dateRange;
request['axisScale'] = data.searchBar.axisScale;
request['plotType'] = data.searchBar.plotType;
request['aggregate'] = data.searchBar.showAggregate;
request['aggregate'] = data.searchBar.aggregate;
request['keyOutside'] = data.searchBar.keyOutside;
var success = function(response){
data.resultView.imageUrl = response.imageUrl;
data.resultView.errorMessage = '';
pauseInvaders();
hideLoadingIcon();
updateImageLink(query);
};
var error = function(e) {
data.resultView.imageUrl = '';
@@ -732,12 +743,36 @@ function sendPlotRequest(query){
else{
data.resultView.errorMessage = "FAILED: " + JSON.parse(e.responseText).message;
}
pauseInvaders();
hideLoadingIcon();
};
data.searchBar.imagelink = '';
postJson("plots", request, success, error);
}
function updateImageLink(query) {
var groupBy = [];
data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}});
var params = {
'query': query,
'groupBy': groupBy,
'splitByKeys.selected': data.searchBar.splitByKeys.selected,
'limitBy.selected': data.searchBar.limitBy.selected,
'limitBy.number': data.searchBar.limitBy.number,
'dateFrom': data.searchBar.dateFrom,
'dateRange': data.searchBar.dateRange,
'axisScale': data.searchBar.axisScale,
'plotType': data.searchBar.plotType,
'aggregate': data.searchBar.aggregate,
'keyOutside': data.searchBar.keyOutside,
'width': $('#result-image').width(),
'height': $('#result-image').height()
};
data.searchBar.imagelink = window.location.origin+ window.location.pathname + "plots?" + jQuery.param(params);
}
function postJson(url, requestData, successCallback, errorCallback) {
@@ -748,8 +783,7 @@ function postJson(url, requestData, successCallback, errorCallback) {
contentType: 'application/json'
})
.done(successCallback)
.fail(errorCallback)
;
.fail(errorCallback);
}
function getJson(url, requestData, successCallback, errorCallback) {