andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

batch entries between TcpIngestor and PerformanceDB

Browse Source 
One bottleneck was the blocking queue used to transport entries
from the listener thread to the ingestor thread.
Reduced the bottleneck by batching entries.
Interestingly the batch size of 100 was better than batch size
of 1000 and better than 10.
This commit is contained in:
Andreas Huber
2018-12-21 13:11:35 +01:00
parent 73ad27ab96
commit d95a71e32e
5 changed files with 121 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
performanceDb/src/main/java/org/lucares/performance/db/EntryToEntriesIterator.java Normal file
View File

@@ -0,0 +1,26 @@
package org.lucares.performance.db;
import java.util.Iterator;
import org.lucares.pdb.api.Entries;
import org.lucares.pdb.api.Entry;
public class EntryToEntriesIterator implements Iterator<Entries> {
private final Iterator<Entry> entryIterator;
public EntryToEntriesIterator(final Iterator<Entry> entryIterator) {
this.entryIterator = entryIterator;
}
@Override
public boolean hasNext() {
return entryIterator.hasNext();
}
@Override
public Entries next() {
return new Entries(entryIterator.next());
}
}

63
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -12,6 +12,7 @@ import java.util.SortedSet;
import java.util.stream.Stream;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.Entries;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
@@ -37,21 +38,22 @@ public class PerformanceDb implements AutoCloseable {
tagsToFile = new TagsToFile(dataStore);
}
public void putEntry(final Entry entry) throws WriteException {
void putEntry(final Entry entry) throws WriteException {
putEntries(Arrays.asList(entry));
}
public void putEntries(final Iterable<Entry> entries) throws WriteException {
void putEntries(final Iterable<Entry> entries) throws WriteException {
putEntries(entries.iterator());
}
public void putEntries(final Iterator<Entry> entries) throws WriteException {
private void putEntries(final Iterator<Entry> entries) throws WriteException {
final BlockingIteratorIterator<Entry> iterator = new BlockingIteratorIterator<>(entries);
final EntryToEntriesIterator entriesIterator = new EntryToEntriesIterator(entries);
final BlockingIteratorIterator<Entries> iterator = new BlockingIteratorIterator<>(entriesIterator);
putEntries(iterator);
}
public void putEntries(final BlockingIterator<Entry> entries) throws WriteException {
public void putEntries(final BlockingIterator<Entries> entriesIterator) throws WriteException {
final Duration timeBetweenSyncs = Duration.ofSeconds(1);
long count = 0;
@@ -62,39 +64,42 @@ public class PerformanceDb implements AutoCloseable {
long nextSync = lastSync + timeBetweenSyncs.toMillis();
while (true) {
final Optional<Entry> entryOptional = entries.next();
if (!entryOptional.isPresent()) {
final Optional<Entries> entriesOptional = entriesIterator.next();
if (!entriesOptional.isPresent()) {
break;
}
final Entry entry = entryOptional.get();
try {
final Tags tags = entry.getTags();
final long dateAsEpochMilli = entry.getEpochMilli();
final Entries entries = entriesOptional.get();
for (final Entry entry : entries) {
final PdbWriter writer = tagsToFile.getWriter(dateAsEpochMilli, tags);
try {
final Tags tags = entry.getTags();
final long dateAsEpochMilli = entry.getEpochMilli();
writer.write(entry);
count++;
insertionsSinceLastSync++;
final PdbWriter writer = tagsToFile.getWriter(dateAsEpochMilli, tags);
if (nextSync <= System.currentTimeMillis()) {
final long end = System.currentTimeMillis();
final long duration = end - lastSync;
final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
writer.write(entry);
count++;
insertionsSinceLastSync++;
METRICS_LOGGER.debug(
String.format("inserting %d/s ; total: %,d; last: %s", entriesPerSecond, count, entry));
if (nextSync <= System.currentTimeMillis()) {
final long end = System.currentTimeMillis();
final long duration = end - lastSync;
final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
lastSync = System.currentTimeMillis();
nextSync = lastSync + timeBetweenSyncs.toMillis();
insertionsSinceLastSync = 0;
METRICS_LOGGER.debug(String.format("inserting %d/s ; total: %,d; last: %s",
entriesPerSecond, count, entry));
lastSync = System.currentTimeMillis();
nextSync = lastSync + timeBetweenSyncs.toMillis();
insertionsSinceLastSync = 0;
}
} catch (final InvalidValueException | SyntaxException e) {
LOGGER.info("skipping entry: " + e.getMessage() + " : " + entry);
LOGGER.info("", e);
}
} catch (final InvalidValueException | SyntaxException e) {
LOGGER.info("skipping entry: " + e.getMessage() + " : " + entry);
LOGGER.info("", e);
}
}