From fa4921fcc91ca1a62968710d533b3533ed2257a6 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 13 Dec 2016 18:41:19 +0100 Subject: [PATCH] use custom csv writer for performance --- .../org/lucares/performance/db/PdbFile.java | 5 +- .../performance/db/PdbFileIterator.java | 8 ++- .../org/lucares/performance/db/PdbReader.java | 62 +++++-------------- .../org/lucares/recommind/logs/Gnuplot.java | 7 ++- .../org/lucares/recommind/logs/Plotter.java | 24 ++++--- 5 files changed, 43 insertions(+), 63 deletions(-) diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java index 010ff44..a414193 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java @@ -9,10 +9,13 @@ class PdbFile { private final File file; + private final long offsetInEpochMilli; + public PdbFile(final Day day, final File file, final Tags tags) { this.day = day; this.file = file; this.tags = tags; + offsetInEpochMilli = day.getOffsetInEpochMilli(); } public static PdbFile today(final File file, final Tags tags) { @@ -38,7 +41,7 @@ class PdbFile { } public long getOffsetInEpochMilli() { - return getTimeRange().getFrom().toInstant().toEpochMilli(); + return offsetInEpochMilli; } @Override diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index e59c174..8b66dac 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -34,9 +34,9 @@ public class PdbFileIterator implements Iterator, AutoCloseable { if (reader == null) { return null; } - final Optional optionalEntry = reader.readEntry(currentPdbFile.getTags()); + final Entry entry = reader.readNullableEntry(currentPdbFile.getTags()); - return optionalEntry.orElseGet(() -> { + if (entry == null) { nextFile(); if (reader == null) { return null; @@ -44,7 +44,9 @@ public class PdbFileIterator implements Iterator, AutoCloseable { final Tags tags = currentPdbFile.getTags(); return reader.readEntry(tags).orElse(null); } - }); + } + + return entry; } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index 711c5c4..511347d 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -30,8 +30,6 @@ class PdbReader implements AutoCloseable { * @return the value or -1 if end of stream has been reached */ public long readValue() { - assertPositionIsAValuePosition(); - return read(); } @@ -42,13 +40,11 @@ class PdbReader implements AutoCloseable { * @throws IOException */ public long readEpochMilli() { - assertPositionIsADatePosition(); - final long value = read(); if (value < 0) { return -1; } - return pdbFile.getDay().getOffsetInEpochMilli() + value; + return pdbFile.getOffsetInEpochMilli() + value; } public OffsetDateTime readDate() { @@ -65,11 +61,12 @@ class PdbReader implements AutoCloseable { try { final int read = data.read(buffer); - if (read < 0) { - return -1; - } if (read != BYTES_PER_VALUE) { - throw new IllegalStateException("invalid file"); + if (read < 0) { + return -1; + } else { + throw new IllegalStateException("invalid file"); + } } return BitFiddling.makeLong(buffer[0], buffer[1], buffer[2], buffer[3]); } catch (final IOException e) { @@ -77,40 +74,6 @@ class PdbReader implements AutoCloseable { } } - private void assertPositionIsADatePosition() { - try { - assertPositionIsValid(); - - if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 0) { - throw new IllegalStateException("file pointer is not at a date position: " + data.getFilePointer()); - } - } catch (final IOException e) { - throw new ReadRuntimeException(e); - } - } - - private void assertPositionIsAValuePosition() { - assertPositionIsValid(); - try { - if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 1) { - throw new IllegalStateException("file pointer is not at a value position: " + data.getFilePointer()); - } - } catch (final IOException e) { - throw new ReadRuntimeException(e); - } - } - - private void assertPositionIsValid() { - try { - if (data.getFilePointer() % BYTES_PER_VALUE != 0) { - throw new IllegalStateException("file pointer is at an illegal position. It is at " - + data.getFilePointer() + " which is not divisible by " + BYTES_PER_VALUE); - } - } catch (final IOException e) { - throw new ReadRuntimeException(e); - } - } - /** * Seek to the n-th value. * @@ -183,17 +146,22 @@ class PdbReader implements AutoCloseable { } } - public Optional readEntry(final Tags tags) throws ReadRuntimeException { + Entry readNullableEntry(final Tags tags) throws ReadRuntimeException { final long epochMilli = readEpochMilli(); if (epochMilli < 0) { - return Optional.empty(); + return null; } final long value = readValue(); if (value < 0) { - return Optional.empty(); + return null; } - return Optional.of(new Entry(epochMilli, value, tags)); + return new Entry(epochMilli, value, tags); + } + + public Optional readEntry(final Tags tags) throws ReadRuntimeException { + + return Optional.ofNullable(readNullableEntry(tags)); } } diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java index 540294c..6dc92bd 100644 --- a/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java @@ -27,10 +27,13 @@ public class Gnuplot { final File gnuplotFile = File.createTempFile("gnuplot", ".dem", tmpDirectory.toFile()); Files.write(gnuplotFileContent, gnuplotFile, StandardCharsets.UTF_8); + final long start = System.nanoTime(); + final ProcessBuilder processBuilder = new ProcessBuilder("gnuplot", gnuplotFile.getAbsolutePath()); processBuilder.inheritIO(); - final Process start = processBuilder.start(); - start.waitFor(); + final Process process = processBuilder.start(); + process.waitFor(); + System.out.println("gnuplot: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); } } diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java index f8d3958..f742adc 100644 --- a/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java @@ -3,6 +3,9 @@ package org.lucares.recommind.logs; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -15,10 +18,6 @@ import java.util.stream.Stream; import org.lucares.performance.db.Entry; import org.lucares.performance.db.FileUtils; import org.lucares.performance.db.PerformanceDb; -import org.lucares.tanga.svak.StreamSvWriter; -import org.lucares.tanga.svak.StreamSvWriterSettings; -import org.lucares.tanga.svak.StreamSvWriterSettings.Escaping; -import org.lucares.tanga.svak.StreamSvWriterSettings.FieldQuoting; public class Plotter { public static void main(final String[] args) throws Exception { @@ -54,11 +53,11 @@ public class Plotter { private static void toCsv(final Stream entries, final File dataFile) throws IOException { - final StreamSvWriterSettings svSettings = new StreamSvWriterSettings(",", "'"); - svSettings.setFieldQuoting(FieldQuoting.IF_NEEDED); - svSettings.setEscaping(Escaping.NONE); - try (FileOutputStream output = new FileOutputStream(dataFile); - StreamSvWriter writer = new StreamSvWriter(output, svSettings)) { + final long start = System.nanoTime(); + int count = 0; + final int separator = ','; + final int newline = '\n'; + try (final Writer output = new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII);) { final Iterator it = entries.iterator(); while (it.hasNext()) { @@ -66,8 +65,13 @@ public class Plotter { final String value = String.valueOf(entry.getValue()); final String date = entry.getDate().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - writer.writeLine(date, value); + output.write(date); + output.write(separator); + output.write(value); + output.write(newline); + count++; } } + System.out.println("wrote " + count + " values to csv in: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); } }