minor unimportant changes

This commit is contained in:
2016-12-30 13:16:30 +01:00
parent 58bb64c80a
commit 62437f384f
4 changed files with 9 additions and 7 deletions

View File

@@ -155,7 +155,7 @@ public class TcpIngestor implements AutoCloseable {
public void start() throws Exception { public void start() throws Exception {
final ArrayBlockingQueue<Entry> queue = new ArrayBlockingQueue<>(100); final ArrayBlockingQueue<Entry> queue = new ArrayBlockingQueue<>(1);
serverThreadPool.submit(() -> { serverThreadPool.submit(() -> {
Thread.currentThread().setName("db-ingestion"); Thread.currentThread().setName("db-ingestion");

View File

@@ -57,7 +57,7 @@ public class PerformanceDb implements AutoCloseable {
public void put(final BlockingIterator<Entry> entries) throws WriteException { public void put(final BlockingIterator<Entry> entries) throws WriteException {
final int blocksize = 10000; final int blocksize = 100000;
long count = 0; long count = 0;
try { try {
@@ -83,7 +83,8 @@ public class PerformanceDb implements AutoCloseable {
final long end = System.nanoTime(); final long end = System.nanoTime();
final double duration = (end - start) / 1_000_000.0; final double duration = (end - start) / 1_000_000.0;
LOGGER.info("inserting the last " + blocksize + " took " + duration + " ms; " + Stats.duration LOGGER.info("inserting the last " + blocksize + " took " + duration + " ms; " + Stats.duration
+ "ms of " + Stats.count + " operations. total entries: " + count); + "ms of " + Stats.count + " operations. total entries: " + count / 1_000_000.0
+ " million");
// System.out.println(entry); // System.out.println(entry);
@@ -91,6 +92,10 @@ public class PerformanceDb implements AutoCloseable {
Stats.duration = 0.0; Stats.duration = 0.0;
Stats.count = 0; Stats.count = 0;
} }
if (count % blocksize == 0) {
tagsToFile.flush();
}
} }
} catch (final RuntimeException e) { } catch (final RuntimeException e) {

View File

@@ -9,5 +9,6 @@ public class Stats {
final long s = System.nanoTime(); final long s = System.nanoTime();
final long e = System.nanoTime(); final long e = System.nanoTime();
Stats.duration += (e - s) / 1_000_000.0; Stats.duration += (e - s) / 1_000_000.0;
Stats.count++;
} }
} }

View File

@@ -161,15 +161,11 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
final PdbWriter result; final PdbWriter result;
final WriterCache writersForTags = getOrInit(tags); final WriterCache writersForTags = getOrInit(tags);
final long s = System.nanoTime();
final Optional<PdbWriter> optionalWriter = chooseBestMatchingWriter(writersForTags.getWriters(), date); final Optional<PdbWriter> optionalWriter = chooseBestMatchingWriter(writersForTags.getWriters(), date);
final long e = System.nanoTime();
Stats.duration += (e - s) / 1_000_000.0;
if (optionalWriter.isPresent()) { if (optionalWriter.isPresent()) {
result = optionalWriter.get(); result = optionalWriter.get();
} else { } else {
Stats.count++;
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags); final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
assertAllFilesHaveSameFolder(pdbFiles); assertAllFilesHaveSameFolder(pdbFiles);