andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use custom csv writer for performance

Browse Source
This commit is contained in:
Andreas Huber
2016-12-13 18:41:19 +01:00
parent 876520eb4c
commit fa4921fcc9
5 changed files with 43 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -9,10 +9,13 @@ class PdbFile {
private final File file;
private final long offsetInEpochMilli;
public PdbFile(final Day day, final File file, final Tags tags) {
this.day = day;
this.file = file;
this.tags = tags;
offsetInEpochMilli = day.getOffsetInEpochMilli();
}
public static PdbFile today(final File file, final Tags tags) {
@@ -38,7 +41,7 @@ class PdbFile {
}
public long getOffsetInEpochMilli() {
return getTimeRange().getFrom().toInstant().toEpochMilli();
return offsetInEpochMilli;
}
@Override

8
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -34,9 +34,9 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader == null) {
return null;
}
final Optional<Entry> optionalEntry = reader.readEntry(currentPdbFile.getTags());
final Entry entry = reader.readNullableEntry(currentPdbFile.getTags());
return optionalEntry.orElseGet(() -> {
if (entry == null) {
nextFile();
if (reader == null) {
return null;
@@ -44,7 +44,9 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
final Tags tags = currentPdbFile.getTags();
return reader.readEntry(tags).orElse(null);
}
});
}
return entry;
}

62
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -30,8 +30,6 @@ class PdbReader implements AutoCloseable {
* @return the value or -1 if end of stream has been reached
*/
public long readValue() {
assertPositionIsAValuePosition();
return read();
}
@@ -42,13 +40,11 @@ class PdbReader implements AutoCloseable {
* @throws IOException
*/
public long readEpochMilli() {
assertPositionIsADatePosition();
final long value = read();
if (value < 0) {
return -1;
}
return pdbFile.getDay().getOffsetInEpochMilli() + value;
return pdbFile.getOffsetInEpochMilli() + value;
}
public OffsetDateTime readDate() {
@@ -65,11 +61,12 @@ class PdbReader implements AutoCloseable {
try {
final int read = data.read(buffer);
if (read < 0) {
return -1;
}
if (read != BYTES_PER_VALUE) {
throw new IllegalStateException("invalid file");
if (read < 0) {
return -1;
} else {
throw new IllegalStateException("invalid file");
}
}
return BitFiddling.makeLong(buffer[0], buffer[1], buffer[2], buffer[3]);
} catch (final IOException e) {
@@ -77,40 +74,6 @@ class PdbReader implements AutoCloseable {
}
}
private void assertPositionIsADatePosition() {
try {
assertPositionIsValid();
if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 0) {
throw new IllegalStateException("file pointer is not at a date position: " + data.getFilePointer());
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private void assertPositionIsAValuePosition() {
assertPositionIsValid();
try {
if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 1) {
throw new IllegalStateException("file pointer is not at a value position: " + data.getFilePointer());
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private void assertPositionIsValid() {
try {
if (data.getFilePointer() % BYTES_PER_VALUE != 0) {
throw new IllegalStateException("file pointer is at an illegal position. It is at "
+ data.getFilePointer() + " which is not divisible by " + BYTES_PER_VALUE);
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
/**
* Seek to the n-th value.
*
@@ -183,17 +146,22 @@ class PdbReader implements AutoCloseable {
}
}
public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
Entry readNullableEntry(final Tags tags) throws ReadRuntimeException {
final long epochMilli = readEpochMilli();
if (epochMilli < 0) {
return Optional.empty();
return null;
}
final long value = readValue();
if (value < 0) {
return Optional.empty();
return null;
}
return Optional.of(new Entry(epochMilli, value, tags));
return new Entry(epochMilli, value, tags);
}
public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
return Optional.ofNullable(readNullableEntry(tags));
}
}

7
recommind-logs/src/main/java/org/lucares/recommind/logs/Gnuplot.java
View File

@@ -27,10 +27,13 @@ public class Gnuplot {
final File gnuplotFile = File.createTempFile("gnuplot", ".dem", tmpDirectory.toFile());
Files.write(gnuplotFileContent, gnuplotFile, StandardCharsets.UTF_8);
final long start = System.nanoTime();
final ProcessBuilder processBuilder = new ProcessBuilder("gnuplot", gnuplotFile.getAbsolutePath());
processBuilder.inheritIO();
final Process start = processBuilder.start();
start.waitFor();
final Process process = processBuilder.start();
process.waitFor();
System.out.println("gnuplot: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}

24
recommind-logs/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -3,6 +3,9 @@ package org.lucares.recommind.logs;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -15,10 +18,6 @@ import java.util.stream.Stream;
import org.lucares.performance.db.Entry;
import org.lucares.performance.db.FileUtils;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.tanga.svak.StreamSvWriter;
import org.lucares.tanga.svak.StreamSvWriterSettings;
import org.lucares.tanga.svak.StreamSvWriterSettings.Escaping;
import org.lucares.tanga.svak.StreamSvWriterSettings.FieldQuoting;
public class Plotter {
public static void main(final String[] args) throws Exception {
@@ -54,11 +53,11 @@ public class Plotter {
private static void toCsv(final Stream<Entry> entries, final File dataFile) throws IOException {
final StreamSvWriterSettings svSettings = new StreamSvWriterSettings(",", "'");
svSettings.setFieldQuoting(FieldQuoting.IF_NEEDED);
svSettings.setEscaping(Escaping.NONE);
try (FileOutputStream output = new FileOutputStream(dataFile);
StreamSvWriter writer = new StreamSvWriter(output, svSettings)) {
final long start = System.nanoTime();
int count = 0;
final int separator = ',';
final int newline = '\n';
try (final Writer output = new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII);) {
final Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
@@ -66,8 +65,13 @@ public class Plotter {
final String value = String.valueOf(entry.getValue());
final String date = entry.getDate().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
writer.writeLine(date, value);
output.write(date);
output.write(separator);
output.write(value);
output.write(newline);
count++;
}
}
System.out.println("wrote " + count + " values to csv in: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}