andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use unix timestamps in the CSVs used by gnuplot

Browse Source 
This is faster by factor 4, because we don't have to format the date.
This commit is contained in:
Andreas Huber
2017-09-24 09:38:08 +02:00
parent 955127dc4a
commit 37eb05aebf
2 changed files with 23 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotSettings.java
View File

@@ -9,7 +9,7 @@ public class GnuplotSettings {
private String terminal = "png"; private String terminal = "png";
private int height = 1200; private int height = 1200;
private int width = 1600; private int width = 1600;
private String timefmt = "%Y-%m-%dT%H:%M:%S"; private String timefmt = "%s"; //"%Y-%m-%dT%H:%M:%S"; // TODO @ahr timefmt
// set format for x-axis // set format for x-axis
private String formatX = "%Y-%m-%d %H:%M:%S"; private String formatX = "%Y-%m-%d %H:%M:%S";

30
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -15,9 +15,11 @@ import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Formatter;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Entry;
@@ -130,15 +132,15 @@ public class Plotter {
if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) { if (minDate.until(maxDate, ChronoUnit.WEEKS) > 1) {
formatX = "%Y-%m-%d"; formatX = "%Y-%m-%d";
rotateX = 0; rotateX = 0;
} else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 10) { } else if (minDate.until(maxDate, ChronoUnit.SECONDS) > 30) {
formatX = "%Y-%m-%d %H:%M:%S"; formatX = "%Y-%m-%d %H:%M:%S";
rotateX = gnuplotSettings.getRotateXAxisLabel(); rotateX = gnuplotSettings.getRotateXAxisLabel();
} else { } else {
formatX = "%Y-%m-%d %H:%M:%.3S"; formatX = "%Y-%m-%d %H:%M:%.3S";
rotateX = gnuplotSettings.getRotateXAxisLabel(); rotateX = gnuplotSettings.getRotateXAxisLabel();
} }
formattedMinDate = minDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")); formattedMinDate = String.valueOf(minDate.toEpochSecond());
formattedMaxDate = maxDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss")); formattedMaxDate = String.valueOf(maxDate.toEpochSecond());
gnuplotSettings.setFormatX(formatX); gnuplotSettings.setFormatX(formatX);
gnuplotSettings.setRotateXAxisLabel(rotateX); gnuplotSettings.setRotateXAxisLabel(rotateX);
@@ -216,21 +218,33 @@ public class Plotter {
int count = 0; int count = 0;
final long fromEpochMilli = dateFrom.toInstant().toEpochMilli(); final long fromEpochMilli = dateFrom.toInstant().toEpochMilli();
final long toEpochMilli = dateTo.toInstant().toEpochMilli(); final long toEpochMilli = dateTo.toInstant().toEpochMilli();
final boolean useMillis = (toEpochMilli - fromEpochMilli) < TimeUnit.MINUTES.toMillis(5);
long maxValue = 0; long maxValue = 0;
long ignoredValues = 0; long ignoredValues = 0;
final int separator = ','; final int separator = ',';
final int newline = '\n'; final int newline = '\n';
final DateTimeFormatter formatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; final StringBuilder formattedDateBuilder = new StringBuilder();
try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { try (final Writer output = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));
final Formatter formatter = new Formatter(formattedDateBuilder);) {
final Iterator<Entry> it = entries.iterator(); final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) { while (it.hasNext()) {
final Entry entry = it.next(); final Entry entry = it.next();
if (fromEpochMilli <= entry.getEpochMilli() && entry.getEpochMilli() <= toEpochMilli) { if (fromEpochMilli <= entry.getEpochMilli() && entry.getEpochMilli() <= toEpochMilli) {
final OffsetDateTime date = entry.getDate();
final String value = String.valueOf(entry.getValue()); final String value = String.valueOf(entry.getValue());
final String formattedDate = date.format(formatter); final String formattedDate;
if (useMillis){
formattedDateBuilder.delete(0, formattedDateBuilder.length());
formatter.format("%.3f", entry.getEpochMilli() / 1000.0);
formattedDate = formattedDateBuilder.toString();
}else {
formattedDate = String.valueOf(entry.getEpochMilli() / 1000);
}
output.write(formattedDate); output.write(formattedDate);
output.write(separator); output.write(separator);
output.write(value); output.write(value);
@@ -244,7 +258,7 @@ public class Plotter {
} }
} }
METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values)", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues); METRICS_LOGGER.debug("wrote {} values to csv in: {}ms (ignored {} values) use millis: {}", count, (System.nanoTime() - start) / 1_000_000.0, ignoredValues, Boolean.toString(useMillis));
return new CsvSummary(count, maxValue); return new CsvSummary(count, maxValue);
} }
} }