diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java index c6e0845..4529efe 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/PercentilePlot.java @@ -29,24 +29,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class PercentilePlot implements ConcretePlotter { - private static final Logger LOGGER = LoggerFactory - .getLogger(ScatterPlot.class); - private static final Logger METRICS_LOGGER = LoggerFactory - .getLogger("org.lucares.metrics.plotter.percentile"); - private PerformanceDb db; - private Path tmpBaseDir; - private Path outputDir; + private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class); + private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.percentile"); + private final PerformanceDb db; + private final Path tmpBaseDir; + private final Path outputDir; - public PercentilePlot(PerformanceDb db, final Path tmpBaseDir, - Path outputDir) { + public PercentilePlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) { this.db = db; this.tmpBaseDir = tmpBaseDir; this.outputDir = outputDir; } @Override - public PlotResult plot(PlotSettings plotSettings) - throws InternalPlottingException { + public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException { LOGGER.trace("start plot: {}", plotSettings); @@ -54,8 +50,7 @@ public class PercentilePlot implements ConcretePlotter { final Path tmpDir = tmpBaseDir.resolve(tmpSubDir); try { Files.createDirectories(tmpDir); - final List dataSeries = Collections - .synchronizedList(new ArrayList<>()); + final List dataSeries = Collections.synchronizedList(new ArrayList<>()); final String query = plotSettings.getQuery(); final List groupBy = plotSettings.getGroupBy(); @@ -68,43 +63,35 @@ public class PercentilePlot implements ConcretePlotter { final long start = System.nanoTime(); final AtomicInteger idCounter = new AtomicInteger(0); - result.getGroups() - .stream() - .parallel() - .forEach( - groupResult -> { - try { - final int id = idCounter.getAndIncrement(); + result.getGroups().stream().parallel().forEach(groupResult -> { + try { + final int id = idCounter.getAndIncrement(); - final FileBackedDataSeries dataSerie = toCsv( - id, groupResult, tmpDir, dateFrom, - dateTo, plotSettings); + final FileBackedDataSeries dataSerie = toCsv(id, groupResult, tmpDir, dateFrom, dateTo, + plotSettings); - if (dataSerie.getValues() > 0) { - dataSeries.add(dataSerie); - } - } catch (Exception e) { - throw new IllegalStateException(e); // TODO - // handle - } - }); - METRICS_LOGGER.debug("csv generation took: " - + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + if (dataSerie.getValues() > 0) { + dataSeries.add(dataSerie); + } + } catch (final Exception e) { + throw new IllegalStateException(e); // TODO + // handle + } + }); + METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); if (dataSeries.isEmpty()) { throw new NoDataPointsException(); } final Limit limitBy = plotSettings.getLimitBy(); - int limit = plotSettings.getLimit(); + final int limit = plotSettings.getLimit(); DataSeries.sortAndLimit(dataSeries, limitBy, limit); DataSeries.setColors(dataSeries); - final Path outputFile = Files.createTempFile(outputDir, "out", - ".png"); + final Path outputFile = Files.createTempFile(outputDir, "out", ".png"); final Gnuplot gnuplot = new Gnuplot(tmpBaseDir); - final GnuplotSettings gnuplotSettings = new GnuplotSettings( - outputFile); + final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile); gnuplotSettings.setHeight(height); gnuplotSettings.setWidth(width); defineXAxis(gnuplotSettings); @@ -114,22 +101,21 @@ public class PercentilePlot implements ConcretePlotter { gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); gnuplot.plot(gnuplotSettings, dataSeries); - return new PlotResult(outputFile.getFileName(), dataSeries); + return new PlotResult(outputFile, dataSeries); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); throw new IllegalStateException("Plotting was interrupted."); } catch (final IOException e) { - throw new InternalPlottingException("Plotting failed: " - + e.getMessage(), e); + throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e); } finally { FileUtils.delete(tmpDir); LOGGER.trace("done plot"); } } - private FileBackedDataSeries toCsv(int id, GroupResult groupResult, - Path tmpDir, OffsetDateTime dateFrom, OffsetDateTime dateTo, - PlotSettings plotSettings) throws IOException { + private FileBackedDataSeries toCsv(final int id, final GroupResult groupResult, final Path tmpDir, + final OffsetDateTime dateFrom, final OffsetDateTime dateTo, final PlotSettings plotSettings) + throws IOException { final long start = System.nanoTime(); final Stream entries = groupResult.asStream(); @@ -143,13 +129,11 @@ public class PercentilePlot implements ConcretePlotter { final IntList values = new IntList(); // TODO should be a LongList long maxValue = 0; - - final Iterator it = entries.iterator(); while (it.hasNext()) { final Entry entry = it.next(); - long epochMilli = entry.getEpochMilli(); + final long epochMilli = entry.getEpochMilli(); if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) { final long value = entry.getValue(); @@ -160,20 +144,19 @@ public class PercentilePlot implements ConcretePlotter { } } - values.parallelSort(); - + final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); - try(final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));){ - + try (final Writer output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { + final StringBuilder data = new StringBuilder(); if (values.size() > 0) { // compute the percentiles for (int i = 0; i < 100; i++) { data.append(i); data.append(separator); - data.append(values.get((int) Math.floor(values.size() - / 100.0 * i))); + data.append(values.get((int) Math.floor(values.size() / 100.0 * i))); data.append(newline); } maxValue = values.get(values.size() - 1); @@ -184,19 +167,16 @@ public class PercentilePlot implements ConcretePlotter { } output.write(data.toString()); } - METRICS_LOGGER - .debug("wrote {} values to csv in: {}ms (ignored {} values) grouping={}", - count, (System.nanoTime() - start) / 1_000_000.0, - ignoredValues, groupResult.getGroupedBy()); + METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) grouping={}", count, + (System.nanoTime() - start) / 1_000_000.0, ignoredValues, groupResult.getGroupedBy()); - final String title = ConcretePlotter.title(groupResult.getGroupedBy(), - values.size()); + final String title = ConcretePlotter.title(groupResult.getGroupedBy(), values.size()); - CsvSummary csvSummary = new CsvSummary(dataFile, values.size(), maxValue, null); - return new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.LINE); + final CsvSummary csvSummary = new CsvSummary(dataFile, values.size(), maxValue, null); + return new FileBackedDataSeries(id, title, csvSummary, GnuplotLineType.LINE); } - private void defineXAxis(GnuplotSettings gnuplotSettings) { + private void defineXAxis(final GnuplotSettings gnuplotSettings) { final XAxisSettings xAxis = gnuplotSettings.getxAxisSettings(); xAxis.setxDataTime(false); xAxis.setFrom("0"); diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/PlotResult.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/PlotResult.java index 155bc1b..9d59c70 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/PlotResult.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/PlotResult.java @@ -4,17 +4,21 @@ import java.nio.file.Path; import java.util.List; public class PlotResult { - private final Path imageName; + private final Path imagePath; private final List dataSeries; - public PlotResult(final Path imageName, final List dataSeries) { + public PlotResult(final Path imagePath, final List dataSeries) { super(); - this.imageName = imageName; + this.imagePath = imagePath; this.dataSeries = dataSeries; } public Path getImageName() { - return imageName; + return imagePath.getFileName(); + } + + public Path getImagePath() { + return imagePath; } public List getDataSeries() { diff --git a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java index f35da12..4f62530 100644 --- a/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java +++ b/pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java @@ -118,7 +118,7 @@ public class ScatterPlot implements ConcretePlotter { gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside()); gnuplot.plot(gnuplotSettings, dataSeries); - return new PlotResult(outputFile.getFileName(), dataSeries); + return new PlotResult(outputFile, dataSeries); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); throw new IllegalStateException("Plotting was interrupted."); diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/InternalServerError.java b/pdb-ui/src/main/java/org/lucares/pdbui/InternalServerError.java new file mode 100644 index 0000000..1c4cb40 --- /dev/null +++ b/pdb-ui/src/main/java/org/lucares/pdbui/InternalServerError.java @@ -0,0 +1,18 @@ +package org.lucares.pdbui; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR, reason = "Internal Server Error") +public class InternalServerError extends RuntimeException { + + private static final long serialVersionUID = 548651821080252932L; + + public InternalServerError(final String message, final Throwable cause) { + super(message, cause); + } + + public InternalServerError(final Throwable cause) { + super(cause); + } +} diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 94a9390..4d93a10 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -1,5 +1,7 @@ package org.lucares.pdbui; +import java.io.FileInputStream; +import java.io.OutputStream; import java.text.Collator; import java.util.ArrayList; import java.util.Collections; @@ -12,7 +14,11 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.lucares.pdb.datastore.Proposal; +import org.lucares.pdb.plot.api.AxisScale; +import org.lucares.pdb.plot.api.Limit; import org.lucares.pdb.plot.api.PlotSettings; +import org.lucares.pdb.plot.api.PlotType; +import org.lucares.pdbui.domain.Aggregate; import org.lucares.pdbui.domain.AutocompleteProposal; import org.lucares.pdbui.domain.AutocompleteProposalByValue; import org.lucares.pdbui.domain.AutocompleteResponse; @@ -31,6 +37,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.http.MediaType; import org.springframework.stereotype.Controller; +import org.springframework.util.StreamUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -39,6 +46,7 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; @Controller @EnableAutoConfiguration @@ -105,6 +113,63 @@ public class PdbController implements HardcodedValues { } } + @RequestMapping(path = "/plots", // + method = RequestMethod.GET, // + produces = MediaType.APPLICATION_OCTET_STREAM_VALUE // + ) + StreamingResponseBody createPlotImage(@RequestParam(name = "query", defaultValue = "") final String query, + @RequestParam(name = "groupBy[]", defaultValue = "") final List aGroupBy, + @RequestParam(name = "limitBy.number", defaultValue = "10") final int limit, + @RequestParam(name = "limitBy.selected", defaultValue = "NO_LIMIT") final Limit limitBy, + @RequestParam(name = "dateFrom", defaultValue = "") final String dateFrom, + @RequestParam(name = "dateRange", defaultValue = "1 week") final String dateRange, + @RequestParam(name = "axisScale", defaultValue = "LINEAR") final AxisScale axisScale, + @RequestParam(name = "plotType", defaultValue = "SCATTER") final PlotType plotType, + @RequestParam(name = "aggregate", defaultValue = "NONE") final Aggregate aggregate, + @RequestParam(name = "keyOutside", defaultValue = "false") final boolean keyOutside, + @RequestParam(name = "height", defaultValue = "1080") final int height, + @RequestParam(name = "width", defaultValue = "1920") final int hidth) { + return (final OutputStream outputStream) -> { + + if (StringUtils.isBlank(query)) { + throw new BadRequest("The query must not be empty!"); + } + + final PlotSettings plotSettings = new PlotSettings(); + plotSettings.setQuery(query); + plotSettings.setGroupBy(aGroupBy); + plotSettings.setHeight(height); + plotSettings.setWidth(hidth); + plotSettings.setLimit(limit); + plotSettings.setLimitBy(limitBy); + plotSettings.setDateFrom(dateFrom); + plotSettings.setDateRange(dateRange); + plotSettings.setYAxisScale(axisScale); + plotSettings.setPlotType(plotType); + plotSettings.setAggregate(PlotSettingsTransformer.toAggregateInternal(aggregate)); + plotSettings.setKeyOutside(keyOutside); + + if (plotterLock.tryLock()) { + try { + final PlotResult result = plotter.plot(plotSettings); + + try (FileInputStream in = new FileInputStream(result.getImagePath().toFile())) { + StreamUtils.copy(in, outputStream); + } + } catch (final NoDataPointsException e) { + throw new NotFoundException(e); + } catch (final InternalPlottingException e) { + throw new InternalServerError(e); + } finally { + plotterLock.unlock(); + } + + } else { + throw new ServiceUnavailableException("Too many parallel requests!"); + } + }; + } + @RequestMapping(path = "/autocomplete", // method = RequestMethod.GET, // produces = MediaType.APPLICATION_JSON_UTF8_VALUE // diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java index 17a6b97..d44ae21 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java @@ -23,15 +23,17 @@ class PlotSettingsTransformer { result.setYAxisScale(request.getAxisScale()); result.setPlotType(request.getPlotType()); result.setAggregate(toAggregateInternal(request.getAggregate())); - result.setKeyOutside(request.isKeyOutside()); + result.setKeyOutside(request.isKeyOutside()); return result; } - private static AggregateHandler toAggregateInternal(Aggregate aggregate) { + static AggregateHandler toAggregateInternal(final Aggregate aggregate) { switch (aggregate) { - case NONE:return new NullAggregate(); - case PERCENTILES:return new PercentileAggregate(); + case NONE: + return new NullAggregate(); + case PERCENTILES: + return new PercentileAggregate(); } throw new IllegalStateException("unhandled enum: " + aggregate); } diff --git a/pdb-ui/src/main/resources/resources/js/ui.js b/pdb-ui/src/main/resources/resources/js/ui.js index b4739d0..460246f 100644 --- a/pdb-ui/src/main/resources/resources/js/ui.js +++ b/pdb-ui/src/main/resources/resources/js/ui.js @@ -200,7 +200,7 @@ Vue.component('result-view', { }, computed: { showPrevNext: function() { - return data.searchBar.splitBy.values.length > 0 && data.resultView.imageUrl; + return data.searchBar.splitBy.values.length > 0 && !data.resultView.loadingGameActive; } }, template: ` @@ -445,13 +445,12 @@ Vue.component('search-bar', { }, computed: { permalink: function() { - - var groupBy = []; - data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}}); - - var params = { - 'query': data.searchBar.query, - 'groupBy': groupBy, + var groupBy = []; + data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}}); + + var params = { + 'query': data.searchBar.query, + 'groupBy': groupBy, 'splitByKeys.selected': data.searchBar.splitByKeys.selected, 'limitBy.selected': data.searchBar.limitBy.selected, 'limitBy.number': data.searchBar.limitBy.number, @@ -459,12 +458,12 @@ Vue.component('search-bar', { 'dateRange': data.searchBar.dateRange, 'axisScale': data.searchBar.axisScale, 'plotType': data.searchBar.plotType, - 'showAggregate': data.searchBar.showAggregate, + 'aggregate': data.searchBar.aggregate, 'keyOutside': data.searchBar.keyOutside, - }; - - var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params ); - return link; + }; + + var link = window.location.origin+ window.location.pathname + "?" + jQuery.param( params ); + return link; } }, template: ` @@ -554,7 +553,7 @@ Vue.component('search-bar', {
- @@ -578,7 +577,9 @@ Vue.component('search-bar', { v-on:click.prevent.stop="dashboard" > Dashboard --> - + + +
` @@ -638,7 +639,7 @@ var data = { dateRange: GetURLParameter('dateRange','1 week'), axisScale: GetURLParameter('axisScale','LOG10'), plotType: GetURLParameter('plotType','SCATTER'), - showAggregate: GetURLParameter('showAggregate','NONE'), + aggregate: GetURLParameter('aggregate','NONE'), keyOutside: GetURLParameterBoolean('keyOutside', 'false'), splitBy: { @@ -646,12 +647,14 @@ var data = { query: '', values: [], index: 0 - } + }, + imagelink: "" }, resultView: { imageUrl: '', - errorMessage: '' - } + errorMessage: '', + loadingGameActive: false + }, }; @@ -659,10 +662,17 @@ function showLoadingIcon() { data.resultView.imageUrl = ''; data.resultView.errorMessage = ''; + data.resultView.loadingGameActive = true; startInvaders(); } +function hideLoadingIcon() +{ + data.resultView.loadingGameActive = false; + pauseInvaders(); +} + function plotCurrent() { showLoadingIcon(); @@ -712,14 +722,15 @@ function sendPlotRequest(query){ request['dateRange'] = data.searchBar.dateRange; request['axisScale'] = data.searchBar.axisScale; request['plotType'] = data.searchBar.plotType; - request['aggregate'] = data.searchBar.showAggregate; + request['aggregate'] = data.searchBar.aggregate; request['keyOutside'] = data.searchBar.keyOutside; var success = function(response){ data.resultView.imageUrl = response.imageUrl; data.resultView.errorMessage = ''; - pauseInvaders(); + hideLoadingIcon(); + updateImageLink(query); }; var error = function(e) { data.resultView.imageUrl = ''; @@ -732,12 +743,36 @@ function sendPlotRequest(query){ else{ data.resultView.errorMessage = "FAILED: " + JSON.parse(e.responseText).message; } - pauseInvaders(); + hideLoadingIcon(); }; + data.searchBar.imagelink = ''; postJson("plots", request, success, error); } +function updateImageLink(query) { + + var groupBy = []; + data.searchBar.groupByKeys.forEach(function(e){ if (e.selected) {groupBy.push(e.selected);}}); + + var params = { + 'query': query, + 'groupBy': groupBy, + 'splitByKeys.selected': data.searchBar.splitByKeys.selected, + 'limitBy.selected': data.searchBar.limitBy.selected, + 'limitBy.number': data.searchBar.limitBy.number, + 'dateFrom': data.searchBar.dateFrom, + 'dateRange': data.searchBar.dateRange, + 'axisScale': data.searchBar.axisScale, + 'plotType': data.searchBar.plotType, + 'aggregate': data.searchBar.aggregate, + 'keyOutside': data.searchBar.keyOutside, + 'width': $('#result-image').width(), + 'height': $('#result-image').height() + }; + + data.searchBar.imagelink = window.location.origin+ window.location.pathname + "plots?" + jQuery.param(params); +} function postJson(url, requestData, successCallback, errorCallback) { @@ -748,8 +783,7 @@ function postJson(url, requestData, successCallback, errorCallback) { contentType: 'application/json' }) .done(successCallback) - .fail(errorCallback) - ; + .fail(errorCallback); } function getJson(url, requestData, successCallback, errorCallback) {