send CSV file via REST

This commit is contained in:
2019-12-08 18:39:34 +01:00
parent f1ef13c1de
commit 85679ca0c8
12 changed files with 334 additions and 48 deletions

View File

@@ -8,6 +8,10 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.lucares.collections.LongList;
@@ -34,10 +38,36 @@ public class PerformanceDb implements AutoCloseable {
private final static Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.ingestion.block");
private final DataStore dataStore;
private final ExecutorService serverThreadPool = Executors.newFixedThreadPool(1);
private final ArrayBlockingQueue<Entries> queue;
public PerformanceDb(final Path dataDirectory) throws IOException {
queue = new ArrayBlockingQueue<>(10);
dataStore = new DataStore(dataDirectory);
startThread();
}
public ArrayBlockingQueue<Entries> getQueue() {
return queue;
}
private void startThread() {
serverThreadPool.submit(() -> {
Thread.currentThread().setName("db-ingestion");
// TODO move error handling to putEntries
boolean finished = false;
while (!finished) {
try {
putEntries(new BlockingQueueIterator<>(queue, Entries.POISON));
finished = true;
} catch (final Exception e) {
LOGGER.warn("Write to database failed. Will retry with the next element.", e);
}
}
return null;
});
}
@@ -56,7 +86,7 @@ public class PerformanceDb implements AutoCloseable {
putEntries(iterator);
}
public void putEntries(final BlockingIterator<Entries> entriesIterator) throws WriteException {
void putEntries(final BlockingIterator<Entries> entriesIterator) throws WriteException {
final Duration timeBetweenSyncs = Duration.ofSeconds(1);
long count = 0;
@@ -162,6 +192,16 @@ public class PerformanceDb implements AutoCloseable {
@Override
public void close() {
try {
LOGGER.debug("adding poison");
queue.put(Entries.POISON);
serverThreadPool.shutdown();
try {
serverThreadPool.awaitTermination(10, TimeUnit.MINUTES);
} catch (final InterruptedException e) {
Thread.interrupted();
}
dataStore.close();
} catch (final Exception e) {
LOGGER.error("failed to close PerformanceDB", e);