andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

log more information in a more predictable manner when inserting entries

Browse Source
This commit is contained in:
Andreas Huber
2017-04-19 19:32:23 +02:00
parent a99f6a276e
commit dc716f8ac4
2 changed files with 19 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Duration;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@@ -63,11 +64,13 @@ public class PerformanceDb implements AutoCloseable {
public void put(final BlockingIterator<Entry> entries) throws WriteException { public void put(final BlockingIterator<Entry> entries) throws WriteException {
final int blocksize = 100000; final Duration timeBetweenSyncs = Duration.ofSeconds(10);
long count = 0; long count = 0;
long insertionsSinceLastSync = 0;
try { try {
long start = System.currentTimeMillis(); long lastSync = System.currentTimeMillis();
long nextSync = lastSync + timeBetweenSyncs.toMillis();
while (true) { while (true) {
final Optional<Entry> entryOptional = nextEntry(entries); final Optional<Entry> entryOptional = nextEntry(entries);
@@ -84,20 +87,22 @@ public class PerformanceDb implements AutoCloseable {
writer.write(entry); writer.write(entry);
count++; count++;
insertionsSinceLastSync++;
if (count % blocksize == 0) { if (nextSync < System.currentTimeMillis()) {
final long end = System.currentTimeMillis(); final long end = System.currentTimeMillis();
final long duration = end - start; final long duration = end - lastSync;
METRICS_LOGGER.debug("inserting the last " + blocksize + " took " + duration final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
+ " ms; total entries: " + count / 1_000_000.0 + " million");
start = System.currentTimeMillis(); METRICS_LOGGER
} .debug(String.format("inserting %d/s ; the last %,d took %dms; total entries: %,d",
entriesPerSecond, insertionsSinceLastSync, duration, count));
if (count % blocksize == 0) {
final long startFlush = System.currentTimeMillis();
tagsToFile.flush(); tagsToFile.flush();
LOGGER.debug("flushed all files: " + (System.currentTimeMillis() - startFlush) + "ms");
lastSync = System.currentTimeMillis();
nextSync = lastSync + timeBetweenSyncs.toMillis();
insertionsSinceLastSync = 0;
} }
} catch (final InvalidValueException | SyntaxException e) { } catch (final InvalidValueException | SyntaxException e) {

4
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -193,7 +193,7 @@ public class TagsToFile implements AutoCloseable {
} }
public void clearWriterCache() { public void clearWriterCache() {
LOGGER.debug("close all cached writers"); LOGGER.info("close all cached writers");
final Iterator<Entry<Tags, WriterCache>> it = cachedWriters.entrySet().iterator(); final Iterator<Entry<Tags, WriterCache>> it = cachedWriters.entrySet().iterator();
while (it.hasNext()) { while (it.hasNext()) {
@@ -274,6 +274,7 @@ public class TagsToFile implements AutoCloseable {
} }
public void flush() { public void flush() {
final long startFlush = System.currentTimeMillis();
LOGGER.debug("flushing all writers"); LOGGER.debug("flushing all writers");
forEachWriter(t -> { forEachWriter(t -> {
try { try {
@@ -283,5 +284,6 @@ public class TagsToFile implements AutoCloseable {
throw new WriteException(e); throw new WriteException(e);
} }
}); });
LOGGER.debug("flushed all files: " + (System.currentTimeMillis() - startFlush) + "ms");
} }
} }