andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add different plot types

Browse Source 
Step 1: 
Added PlotType enum and a drop down to the UI.
Extracted the code for scatter plots.
This commit is contained in:
ahr
2017-12-29 08:57:34 +01:00
parent 2df66c7b2f
commit cc70f45c12
13 changed files with 465 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -260,10 +260,6 @@ public class DataStore {
final Doc doc = docIdToDoc.get(docId);
if (!doc.getTags().getValue("pod").equals("vadtrans01")){
System.out.println();
}
result.add(doc);
}
}

10
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java
View File

@@ -36,6 +36,8 @@ public class PlotSettings {
private boolean keyOutside;
private PlotType plotType;
public String getQuery() {
return query;
}
@@ -178,4 +180,12 @@ public class PlotSettings {
public boolean isKeyOutside() {
return keyOutside;
}
public void setPlotType(PlotType plotType) {
this.plotType = plotType;
}
public PlotType getPlotType() {
return plotType;
}
}

5
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotType.java Normal file
View File

@@ -0,0 +1,5 @@
package org.lucares.pdb.plot.api;
public enum PlotType {
SCATTER, PERCENTILES
}

34
pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java
View File

@@ -7,6 +7,7 @@ import java.util.List;
import java.util.Map;
import org.lucares.pdb.plot.api.AggregatedData;
import org.lucares.pdb.plot.api.Limit;
public class DataSeries {
public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (
@@ -66,5 +67,38 @@ public class DataSeries {
return result;
}
static Comparator<? super DataSeries> getDataSeriesComparator(final Limit limitBy) {
switch (limitBy) {
case MOST_VALUES:
return DataSeries.BY_NUMBER_OF_VALUES.reversed();
case FEWEST_VALUES:
return DataSeries.BY_NUMBER_OF_VALUES;
case MAX_VALUE:
return DataSeries.BY_MAX_VALUE.reversed();
case MIN_VALUE:
return DataSeries.BY_MAX_VALUE;
case NO_LIMIT:
return DataSeries.BY_NUMBER_OF_VALUES;
}
throw new IllegalStateException("unhandled enum: "+ limitBy);
}
static void sortAndLimit(final List<DataSeries> dataSeries, final Limit limitBy, final int limit) {
dataSeries.sort(DataSeries.getDataSeriesComparator(limitBy));
switch (limitBy) {
case FEWEST_VALUES:
case MOST_VALUES:
case MAX_VALUE:
case MIN_VALUE:
while (dataSeries.size() > limit) {
dataSeries.remove(limit);
}
break;
case NO_LIMIT:
}
}
}

15
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java
View File

@@ -21,11 +21,14 @@ public class GnuplotFileGenerator {
appendfln(result, "set timefmt '%s'", settings.getTimefmt());
final XAxisSettings xAxis = settings.getxAxisSettings();
if (xAxis.isxDataTime()){
appendfln(result, "set xdata time");
appendfln(result, "set format x \"%s\"", settings.getFormatX());
appendfln(result, "set xlabel \"%s\"", settings.getXlabel());
appendfln(result, "set xtics rotate by %d", settings.getRotateXAxisLabel());
appendfln(result, "set xrange [\"%s\":\"%s\"]", settings.getDateFrom(), settings.getDateTo());
}
appendfln(result, "set format x \"%s\"", xAxis.getFormatX());
appendfln(result, "set xlabel \"%s\"", xAxis.getXlabel());
appendfln(result, "set xtics rotate by %d", xAxis.getRotateXAxisLabel());
appendfln(result, "set xrange [\"%s\":\"%s\"]", xAxis.getFrom(), xAxis.getTo());
final long graphOffset = settings.getYAxisScale() == AxisScale.LINEAR ? 0 : 1;
appendfln(result, "set yrange [\""+graphOffset+"\":]");
@@ -46,8 +49,8 @@ public class GnuplotFileGenerator {
appendfln(result, "set output \"%s\"", settings.getOutput().toAbsolutePath().toString().replace("\\", "/"));
// marker lines that show which area will be zoomed
final long minDate = Long.parseLong(settings.getDateFrom());
final long maxDate = Long.parseLong(settings.getDateTo());
final long minDate = Long.parseLong(settings.getxAxisSettings().getFrom());
final long maxDate = Long.parseLong(settings.getxAxisSettings().getTo());
appendfln(result, "set arrow from "+(minDate + (maxDate-minDate)*0.25)+","+graphOffset+" rto graph 0,1 lt 3 lc rgb \"#EEEEEE\" nohead");
appendfln(result, "set arrow from "+(minDate + (maxDate-minDate)*0.75)+","+graphOffset+" rto graph 0,1 lc rgb \"#EEEEEE\" nohead");

57
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java
View File

@@ -9,16 +9,12 @@ public class GnuplotSettings {
private String terminal = "png";
private int height = 1200;
private int width = 1600;
private String timefmt = "%s"; //"%Y-%m-%dT%H:%M:%S"; // TODO @ahr timefmt
private String timefmt = "%s"; // time as unix epoch, but as double
// set format for x-axis
private String formatX = "%Y-%m-%d %H:%M:%S";
// set datafile separator
private String datafileSeparator = ",";
// set xlabel
private String xlabel = "Time";
// set ylabel
private String ylabel = "Duration in ms";
@@ -26,26 +22,30 @@ public class GnuplotSettings {
// set output "datausage.png"
private final Path output;
// set xtics rotate by 10 degree
private int rotateXAxisLabel = -10;
private AxisScale yAxisScale;
private String dateFrom;
private String dateTo;
private AggregateHandler aggregate;
private boolean keyOutside = false;
private XAxisSettings xAxisSettings = new XAxisSettings();
public GnuplotSettings(final Path output) {
this.output = output;
}
public int getRotateXAxisLabel() {
return rotateXAxisLabel;
public XAxisSettings getxAxisSettings() {
return xAxisSettings;
}
public void setRotateXAxisLabel(final int rotateXAxisLabel) {
this.rotateXAxisLabel = rotateXAxisLabel;
public void setxAxisSettings(XAxisSettings xAxisSettings) {
this.xAxisSettings = xAxisSettings;
}
public String getTerminal() {
return terminal;
}
@@ -78,13 +78,6 @@ public class GnuplotSettings {
this.timefmt = timefmt;
}
public String getFormatX() {
return formatX;
}
public void setFormatX(final String formatX) {
this.formatX = formatX;
}
public String getDatafileSeparator() {
return datafileSeparator;
@@ -94,14 +87,6 @@ public class GnuplotSettings {
this.datafileSeparator = datafileSeparator;
}
public String getXlabel() {
return xlabel;
}
public void setXlabel(final String xlabel) {
this.xlabel = xlabel;
}
public String getYlabel() {
return ylabel;
}
@@ -122,22 +107,6 @@ public class GnuplotSettings {
return yAxisScale;
}
public void setDateFrom(final String dateFrom) {
this.dateFrom = dateFrom;
}
public String getDateFrom() {
return dateFrom;
}
public void setDateTo(final String dateTo) {
this.dateTo = dateTo;
}
public String getDateTo() {
return dateTo;
}
public void setAggregate(AggregateHandler aggregate) {
this.aggregate = aggregate;
}

263
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -1,57 +1,14 @@
package org.lucares.recommind.logs;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Formatter;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdb.plot.api.PlotType;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Plotter {
private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter");
private static final String DEFAULT_GROUP = "<none>";
private static final int INT_TO_STRING_CACHE_SIZE= 1000;
private static final String[] INT_TO_STRING;
static {
INT_TO_STRING = new String[INT_TO_STRING_CACHE_SIZE];
for (int i = 0; i < INT_TO_STRING_CACHE_SIZE; i++){
INT_TO_STRING[i] = String.valueOf(i);
}
}
private final PerformanceDb db;
private final Path tmpBaseDir;
@@ -76,226 +33,22 @@ public class Plotter {
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
LOGGER.trace("start plot: {}", plotSettings);
PlotType plotType = plotSettings.getPlotType();
switch (plotType) {
case SCATTER:
final ScatterPlot scatterPlot = new ScatterPlot(db, tmpBaseDir, outputDir);
return scatterPlot.plot(plotSettings);
final String tmpSubDir = uniqueDirectoryName();
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
try {
Files.createDirectories(tmpDir);
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
final String query = plotSettings.getQuery();
final List<String> groupBy = plotSettings.getGroupBy();
final int height = plotSettings.getHeight();
final int width = plotSettings.getWidth();
final OffsetDateTime dateFrom = plotSettings.dateFrom();
final OffsetDateTime dateTo = plotSettings.dateTo();
final Result result = db.get(query, groupBy);
final long start = System.nanoTime();
final AtomicInteger idCounter = new AtomicInteger(0);
result.getGroups().stream().parallel().forEach(groupResult -> {
try{
final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
final int id = idCounter.getAndIncrement();
final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
final DataSeries dataSerie = new DataSeries("id"+id, title, csvSummary);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
default:
throw new UnsupportedOperationException("plot of type " + plotType + " not supported.");
}
}catch (Exception e){
throw new IllegalStateException( e); // TODO handle
}
});
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0
+ "ms");
if (dataSeries.isEmpty()) {
throw new NoDataPointsException();
}
sortAndLimit(dataSeries, plotSettings);
final Path outputFile = Files.createTempFile(outputDir, "out", ".png");
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
gnuplotSettings.setHeight(height);
gnuplotSettings.setWidth(width);
defineXAxis(gnuplotSettings, plotSettings.dateFrom(), plotSettings.dateTo());
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
gnuplotSettings.setAggregate(plotSettings.getAggregate());
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
gnuplot.plot(gnuplotSettings, dataSeries);
return new PlotResult(outputFile.getFileName(), dataSeries);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Plotting was interrupted.");
} catch (final IOException e) {
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
} finally {
FileUtils.delete(tmpDir);
LOGGER.trace("done plot");
}
}
private void defineXAxis(final GnuplotSettings gnuplotSettings, final OffsetDateTime minDate,
final OffsetDateTime maxDate) {
String formatX;
int rotateX;
String formattedMinDate;
String formattedMaxDate;
if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) {
formatX = "%Y-%m-%d";
rotateX = 0;
} else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) {
formatX = "%Y-%m-%d %H:%M:%S";
rotateX = gnuplotSettings.getRotateXAxisLabel();
} else {
formatX = "%Y-%m-%d %H:%M:%.3S";
rotateX = gnuplotSettings.getRotateXAxisLabel();
}
formattedMinDate = String.valueOf(minDate.toEpochSecond());
formattedMaxDate = String.valueOf(maxDate.toEpochSecond());
gnuplotSettings.setFormatX(formatX);
gnuplotSettings.setRotateXAxisLabel(rotateX);
gnuplotSettings.setDateFrom(formattedMinDate);
gnuplotSettings.setDateTo(formattedMaxDate);
}
private String uniqueDirectoryName() {
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
+ UUID.randomUUID().toString();
}
private void sortAndLimit(final List<DataSeries> dataSeries, final PlotSettings plotSettings) {
final Limit limitBy = plotSettings.getLimitBy();
dataSeries.sort(getDataSeriesComparator(limitBy));
switch (limitBy) {
case FEWEST_VALUES:
case MOST_VALUES:
case MAX_VALUE:
case MIN_VALUE:
while (dataSeries.size() > plotSettings.getLimit()) {
dataSeries.remove(plotSettings.getLimit());
}
break;
case NO_LIMIT:
}
}
private Comparator<? super DataSeries> getDataSeriesComparator(final Limit limitBy) {
switch (limitBy) {
case MOST_VALUES:
return DataSeries.BY_NUMBER_OF_VALUES.reversed();
case FEWEST_VALUES:
return DataSeries.BY_NUMBER_OF_VALUES;
case MAX_VALUE:
return DataSeries.BY_MAX_VALUE.reversed();
case MIN_VALUE:
return DataSeries.BY_MAX_VALUE;
case NO_LIMIT:
return DataSeries.BY_NUMBER_OF_VALUES;
}
throw new IllegalStateException("unhandled enum: "+ limitBy);
}
private String title(final Tags tags, final int values) {
final StringBuilder result = new StringBuilder();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append(" (");
result.append(String.format("%,d", values));
result.append(")");
return result.toString();
}
private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom,
final OffsetDateTime dateTo, PlotSettings plotSettings) throws IOException {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
final long start = System.nanoTime();
final Stream<Entry> entries = groupResult.asStream();
int count = 0;
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
long maxValue = 0;
long ignoredValues = 0;
final int separator = ',';
final int newline = '\n';
final StringBuilder formattedDateBuilder = new StringBuilder();
try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));
final Formatter formatter = new Formatter(formattedDateBuilder);) {
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
final Entry entry = it.next();
long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
long value = entry.getValue();
final String stringValue = longToString(value);
final String formattedDate;
if (useMillis){
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
}else {
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(stringValue);
output.write(newline);
aggregator.addValue(epochMilli, value);
count++;
maxValue = Math.max(maxValue, value);
}else {
ignoredValues++;
}
}
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy(),dataFile);
return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData());
}
private static String longToString(final long value){
// using pre-generated strings reduces memory allocation by up to 25%
if (value < INT_TO_STRING_CACHE_SIZE){
return INT_TO_STRING[(int) value];
}
return String.valueOf(value);
}
}

270
pdb-plotting/src/main/java/org/lucares/recommind/logs/ScatterPlot.java Normal file
View File

@@ -0,0 +1,270 @@
package org.lucares.recommind.logs;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Formatter;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.plot.api.CustomAggregator;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ScatterPlot {
private static final Logger LOGGER = LoggerFactory.getLogger(ScatterPlot.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter.scatter");
private static final String DEFAULT_GROUP = "<none>";
private static final int INT_TO_STRING_CACHE_SIZE= 1000;
private static final String[] INT_TO_STRING;
static {
INT_TO_STRING = new String[INT_TO_STRING_CACHE_SIZE];
for (int i = 0; i < INT_TO_STRING_CACHE_SIZE; i++){
INT_TO_STRING[i] = String.valueOf(i);
}
}
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
public ScatterPlot(final PerformanceDb db, final Path tmpBaseDir, final Path outputDir) {
this.db = db;
this.tmpBaseDir = tmpBaseDir;
this.outputDir = outputDir;
if (!Files.isDirectory(tmpBaseDir, LinkOption.NOFOLLOW_LINKS)) {
throw new IllegalArgumentException(tmpBaseDir + " is not a directory");
}
if (!Files.isDirectory(outputDir)) {
throw new IllegalArgumentException(outputDir + " is not a directory");
}
}
public Path getOutputDir() {
return outputDir;
}
public PlotResult plot(final PlotSettings plotSettings) throws InternalPlottingException {
LOGGER.trace("start plot: {}", plotSettings);
final String tmpSubDir = uniqueDirectoryName();
final Path tmpDir = tmpBaseDir.resolve(tmpSubDir);
try {
Files.createDirectories(tmpDir);
final List<DataSeries> dataSeries = Collections.synchronizedList(new ArrayList<>());
final String query = plotSettings.getQuery();
final List<String> groupBy = plotSettings.getGroupBy();
final int height = plotSettings.getHeight();
final int width = plotSettings.getWidth();
final OffsetDateTime dateFrom = plotSettings.dateFrom();
final OffsetDateTime dateTo = plotSettings.dateTo();
final Result result = db.get(query, groupBy);
final long start = System.nanoTime();
final AtomicInteger idCounter = new AtomicInteger(0);
result.getGroups().stream().parallel().forEach(groupResult -> {
try{
final CsvSummary csvSummary = toCsv(groupResult, tmpDir, dateFrom, dateTo, plotSettings);
final int id = idCounter.getAndIncrement();
final String title = title(groupResult.getGroupedBy(), csvSummary.getValues());
final DataSeries dataSerie = new DataSeries("id"+id, title, csvSummary);
if (dataSerie.getValues() > 0) {
dataSeries.add(dataSerie);
}
}catch (Exception e){
throw new IllegalStateException( e); // TODO handle
}
});
METRICS_LOGGER.debug("csv generation took: " + (System.nanoTime() - start) / 1_000_000.0
+ "ms");
if (dataSeries.isEmpty()) {
throw new NoDataPointsException();
}
final Limit limitBy = plotSettings.getLimitBy();
int limit = plotSettings.getLimit();
DataSeries.sortAndLimit(dataSeries, limitBy, limit);
final Path outputFile = Files.createTempFile(outputDir, "out", ".png");
final Gnuplot gnuplot = new Gnuplot(tmpBaseDir);
final GnuplotSettings gnuplotSettings = new GnuplotSettings(outputFile);
gnuplotSettings.setHeight(height);
gnuplotSettings.setWidth(width);
defineXAxis(gnuplotSettings, plotSettings.dateFrom(), plotSettings.dateTo());
gnuplotSettings.setYAxisScale(plotSettings.getYAxisScale());
gnuplotSettings.setAggregate(plotSettings.getAggregate());
gnuplotSettings.setKeyOutside(plotSettings.isKeyOutside());
gnuplot.plot(gnuplotSettings, dataSeries);
return new PlotResult(outputFile.getFileName(), dataSeries);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Plotting was interrupted.");
} catch (final IOException e) {
throw new InternalPlottingException("Plotting failed: " + e.getMessage(), e);
} finally {
FileUtils.delete(tmpDir);
LOGGER.trace("done plot");
}
}
private void defineXAxis(final GnuplotSettings gnuplotSettings, final OffsetDateTime minDate,
final OffsetDateTime maxDate) {
String formatX;
int rotateX;
String formattedMinDate;
String formattedMaxDate;
if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) {
formatX = "%Y-%m-%d";
rotateX = 0;
} else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) {
formatX = "%Y-%m-%d %H:%M:%S";
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
} else {
formatX = "%Y-%m-%d %H:%M:%.3S";
rotateX = gnuplotSettings.getxAxisSettings().getRotateXAxisLabel();
}
formattedMinDate = String.valueOf(minDate.toEpochSecond());
formattedMaxDate = String.valueOf(maxDate.toEpochSecond());
gnuplotSettings.getxAxisSettings().setFormatX(formatX);
gnuplotSettings.getxAxisSettings().setRotateXAxisLabel(rotateX);
gnuplotSettings.getxAxisSettings().setFrom(formattedMinDate);
gnuplotSettings.getxAxisSettings().setTo(formattedMaxDate);
}
private String uniqueDirectoryName() {
return OffsetDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH_mm_ss")) + "_"
+ UUID.randomUUID().toString();
}
private static CsvSummary toCsv(final GroupResult groupResult, final Path tmpDir, final OffsetDateTime dateFrom,
final OffsetDateTime dateTo, PlotSettings plotSettings) throws IOException {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
final long start = System.nanoTime();
final Stream<Entry> entries = groupResult.asStream();
int count = 0;
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
final CustomAggregator aggregator = plotSettings.getAggregate().createCustomAggregator(fromEpochMilli, toEpochMilli);
long maxValue = 0;
long ignoredValues = 0;
final int separator = ',';
final int newline = '\n';
final StringBuilder formattedDateBuilder = new StringBuilder();
try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));
final Formatter formatter = new Formatter(formattedDateBuilder);) {
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
final Entry entry = it.next();
long epochMilli = entry.getEpochMilli();
if (fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli) {
long value = entry.getValue();
final String stringValue = longToString(value);
final String formattedDate;
if (useMillis){
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", epochMilli / 1000.0);
formattedDate = formattedDateBuilder.toString();
}else {
formattedDate = String.valueOf(epochMilli / 1000);
}
output.write(formattedDate);
output.write(separator);
output.write(stringValue);
output.write(newline);
aggregator.addValue(epochMilli, value);
count++;
maxValue = Math.max(maxValue, value);
}else {
ignoredValues++;
}
}
}
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}, grouping={}, file={}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis), groupResult.getGroupedBy(),dataFile);
return new CsvSummary(dataFile, count, maxValue, aggregator.getAggregatedData());
}
private static String longToString(final long value){
// using pre-generated strings reduces memory allocation by up to 25%
if (value < INT_TO_STRING_CACHE_SIZE){
return INT_TO_STRING[(int) value];
}
return String.valueOf(value);
}
private String title(final Tags tags, final int values) {
final StringBuilder result = new StringBuilder();
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append(" (");
result.append(String.format("%,d", values));
result.append(")");
return result.toString();
}
}

76
pdb-plotting/src/main/java/org/lucares/recommind/logs/XAxisSettings.java Normal file
View File

@@ -0,0 +1,76 @@
package org.lucares.recommind.logs;
public class XAxisSettings {
// set xdata time
private boolean xDataTime = true;
// set format for x-axis
private String formatX = "%Y-%m-%d %H:%M:%S";
// set xlabel
private String xlabel = "Time";
// set xtics rotate by 10 degree
private int rotateXAxisLabel = -10;
private String from;
private String to;
public boolean isxDataTime() {
return xDataTime;
}
public void setxDataTime(boolean xDataTime) {
this.xDataTime = xDataTime;
}
public String getFormatX() {
return formatX;
}
public void setFormatX(String formatX) {
this.formatX = formatX;
}
public String getXlabel() {
return xlabel;
}
public void setXlabel(String xlabel) {
this.xlabel = xlabel;
}
public int getRotateXAxisLabel() {
return rotateXAxisLabel;
}
public void setRotateXAxisLabel(int rotateXAxisLabel) {
this.rotateXAxisLabel = rotateXAxisLabel;
}
public String getFrom() {
return from;
}
public void setFrom(String from) {
this.from = from;
}
public String getTo() {
return to;
}
public void setTo(String to) {
this.to = to;
}
@Override
public String toString() {
return "XAxisSettings [xDataTime=" + xDataTime + ", formatX=" + formatX
+ ", xlabel=" + xlabel + ", rotateXAxisLabel="
+ rotateXAxisLabel + ", from=" + from + ", to=" + to + "]";
}
}

1
pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java
View File

@@ -22,6 +22,7 @@ class PlotSettingsTransformer {
result.setDateFrom(request.getDateFrom());
result.setDateRange(request.getDateRange());
result.setYAxisScale(request.getAxisScale());
result.setPlotType(request.getPlotType());
result.setAggregate(toAggregateInternal(request.getAggregate()));
result.setKeyOutside(request.isKeyOutside());

11
pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java
View File

@@ -4,6 +4,7 @@ import java.util.List;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.Limit;
import org.lucares.pdb.plot.api.PlotType;
public class PlotRequest {
private String query;
@@ -24,6 +25,8 @@ public class PlotRequest {
private String dateRange;
private PlotType plotType = PlotType.SCATTER;
private Aggregate aggregate = Aggregate.NONE;
private boolean keyOutside;
@@ -108,6 +111,14 @@ public class PlotRequest {
this.yAxis = yAxis;
}
public PlotType getPlotType() {
return plotType;
}
public void setPlotType(PlotType plotType) {
this.plotType = plotType;
}
public void setAggregate(Aggregate aggregate) {
this.aggregate = aggregate;
}

14
pdb-ui/src/main/resources/resources/js/search.js
View File

@@ -16,6 +16,9 @@ $(document).ready(function(){
$('#search-limit-by').change(updateSearchLimitValue);
disableSplitBy();
$('#plot-type').change(updatePlotType);
updatePlotType();
$('#nav_left').click(dateLeftShift);
$('#nav_left_half').click(dateHalfLeftShift);
$('#nav_right_half').click(dateHalfRightShift);
@@ -252,6 +255,16 @@ function updateSearchLimitValue () {
}
}
function updatePlotType(){
var optionSelected = $('#plot-type').find("option:selected");
var valueSelected = optionSelected.val();
if (valueSelected == "PERCENTILES"){
$('#group-show-aggregate').hide();
}else{
$('#group-show-aggregate').show();
}
}
function enableSplitBy(fieldValues) {
splitBy['field'] = $('#split-by').val();
splitBy['values'] = fieldValues;
@@ -399,6 +412,7 @@ function sendPlotRequest(query){
request['dateFrom'] = $('#search-date-from').val();
request['dateRange'] = $('#search-date-range').val();
request['axisScale'] = $('#search-y-axis-scale').val();
request['plotType'] = $('#plot-type').val();
request['aggregate'] = $('#show-aggregate').val();
request['keyOutside'] = $('#key-outside').is(":checked");

9
pdb-ui/src/main/resources/templates/main.html
View File

@@ -15,8 +15,6 @@
</head>
<body>
<div id="search-input-wrapper">
<input id="search-input" placeholder="field=value and anotherField=anotherValue" data-autocomplete="autocomplete"
data-autocomplete-empty-message="nothing found" />
@@ -70,6 +68,13 @@
</select>
</div>
<div class="group">
<label for="plot-type">Type:</label>
<select id="plot-type">
<option value="SCATTER" selected="selected">Scatter</option>
<option value="PERCENTILES">Percentiles</option>
</select>
</div>
<div class="group" id="group-show-aggregate">
<label for="show-aggregate">Aggregate:</label>
<select id="show-aggregate">
<option value="NONE" selected="selected">-</option>