TcpIngestor that receives a stream of json objects and stores them

performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java

@@ -50,7 +50,7 @@ class PdbWriter implements AutoCloseable {
 
 	private void assertEpochMilliInRange(final long epochMilli) {
 		if (epochMilli < minimalEpochMilli) {
-			LOGGER.warning("epochMilli must not be smaller than " + minimalEpochMilli + ", but was " + epochMilli
+			LOGGER.fine("epochMilli must not be smaller than " + minimalEpochMilli + ", but was " + epochMilli
 					+ ". We'll accept this for now. "
 					+ "Currently there is no code that relies on monotonically increasing date values. "
 					+ "Log4j does not guarantee it either.");
@@ -84,4 +84,8 @@ class PdbWriter implements AutoCloseable {
 		outputStream.flush();
 		outputStream.close();
 	}
+
+	public void flush() throws IOException {
+		outputStream.flush();
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java

@@ -3,7 +3,9 @@ package org.lucares.performance.db;
 import java.io.IOException;
 import java.time.OffsetDateTime;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -61,12 +63,16 @@ public class PdbWriterManager implements AutoCloseable {
 
 	private final PdbWriterSupplier supplier;
 
+	private Day lastDay = new Day(OffsetDateTime.MIN);
+
 	public PdbWriterManager(final PdbWriterSupplier supplier) {
 		this.supplier = supplier;
 	}
 
 	public PdbWriter get(final Tags tags, final OffsetDateTime date) {
 
+		handleDateChange(date);
+
 		final Key key = new Key(tags, date);
 		if (!map.containsKey(key)) {
 			final PdbWriter writer = supplier.supply(tags, date);
@@ -75,19 +81,51 @@ public class PdbWriterManager implements AutoCloseable {
 		return map.get(key);
 	}
 
+	private void handleDateChange(final OffsetDateTime date) {
+
+		final Day day = new Day(date);
+
+		if (!day.equals(lastDay)) {
+			closeFiles();
+			lastDay = day;
+		}
+	}
+
 	public PdbWriter put(final Tags tags, final OffsetDateTime date, final PdbWriter pdbWriter) {
 		final Key key = new Key(tags, date);
 		return map.put(key, pdbWriter);
 	}
 
-	@Override
-	public void close() {
+	public void flush() {
+		LOGGER.info("flushing all files");
 		for (final PdbWriter writer : map.values()) {
 			try {
-				writer.close();
+				writer.flush();
 			} catch (final IOException e) {
 				LOGGER.log(Level.WARNING, e.getMessage(), e);
 			}
 		}
 	}
+
+	@Override
+	public void close() {
+		closeFiles();
+	}
+
+	private void closeFiles() {
+		LOGGER.info("closing all files");
+		final Iterator<Entry<Key, PdbWriter>> it = map.entrySet().iterator();
+
+		while (it.hasNext()) {
+			final Entry<Key, PdbWriter> entry = it.next();
+			final PdbWriter writer = entry.getValue();
+			try {
+				writer.close();
+
+			} catch (final IOException e) {
+				LOGGER.log(Level.WARNING, e.getMessage(), e);
+			}
+			it.remove();
+		}
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -17,14 +17,12 @@ import java.util.stream.StreamSupport;
 
 import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
 
-import liquibase.exception.LiquibaseException;
-
 public class PerformanceDb implements AutoCloseable {
 	private static final Logger LOGGER = Logger.getLogger(PerformanceDb.class.getCanonicalName());
 
 	private final TagsToFile tagsToFile;
 
-	public PerformanceDb(final Path dataDirectory) throws LiquibaseException {
+	public PerformanceDb(final Path dataDirectory) {
 		tagsToFile = new TagsToFile(dataDirectory);
 	}
 
@@ -71,12 +69,14 @@ public class PerformanceDb implements AutoCloseable {
 
 	public void put(final BlockingIterator<Entry> entries) throws WriteException {
 
-		final long start = System.nanoTime();
-		final double timeSpendInWrite = 0.0;
 		long count = 0;
+		double durationInManager = 0;
 
 		try (final PdbWriterManager manager = new PdbWriterManager(new WriterSupplier(tagsToFile));) {
+			long start = System.nanoTime();
+
 			while (true) {
+
 				final Optional<Entry> entryOptional = entries.next();
 				if (!entryOptional.isPresent()) {
 					break;
@@ -90,20 +90,31 @@ public class PerformanceDb implements AutoCloseable {
 
 				writer.write(entry);
 				count++;
+
+				if (count == 10000) {
+					final long end = System.nanoTime();
+					final double duration = (end - start) / 1_000_000.0;
+					LOGGER.info("inserting the last " + count + " took " + duration + " ms; " + durationInManager
+							+ "ms in entries.next ");
+
+					System.out.println(entry);
+
+					start = System.nanoTime();
+					durationInManager = 0.0;
+					count = 0;
+				}
 			}
 
 		} catch (final InterruptedException e) {
 			Thread.currentThread().interrupt();
 			LOGGER.info("Thread was interrupted. Aborting exectution.");
 		} finally {
-			final double duration = (System.nanoTime() - start) / 1_000_000.0;
-			LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite
-					+ " were spend in write");
+			//
 		}
 	}
 
-	public List<Entry> getAsList(final Tags tags) {
-		return get(tags).collect(Collectors.toList());
+	public List<Entry> getAsList(final String query) {
+		return get(query).collect(Collectors.toList());
 	}
 
 	/**
@@ -138,7 +149,7 @@ public class PerformanceDb implements AutoCloseable {
 	}
 
 	@Override
-	public void close() throws Exception {
+	public void close() {
 		tagsToFile.close();
 	}
 }

performanceDb/src/main/java/org/lucares/performance/db/Tags.java

@@ -117,7 +117,7 @@ public class Tags {
 
 	public String abbreviatedRepresentation() {
 		final StringBuilder result = new StringBuilder();
-		final int maxLength = 500;
+		final int maxLength = 200;
 
 		final SortedSet<String> keys = new TreeSet<>(tags.keySet());
 

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -172,8 +172,13 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 	}
 
 	@Override
-	public void close() throws Exception {
-		db.close();
+	public void close() {
+		try {
+			db.close();
+		} catch (final Exception e) {
+			// H2 doesn't actually do anything in close
+			throw new IllegalStateException(e);
+		}
 	}
 
 }

performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java

@@ -100,7 +100,7 @@ public class PerformanceDbTest {
 				performanceDb.put(entry);
 			}
 
-			final List<Entry> actualEntries = performanceDb.getAsList(tags);
+			final List<Entry> actualEntries = performanceDb.getAsList(Query.createQuery(tags));
 			Assert.assertEquals(actualEntries, entries);
 
 			final File storageFileForToday = StorageUtils.createStorageFile(dataDirectory, new Day(timeRange.getFrom()),
@@ -137,16 +137,16 @@ public class PerformanceDbTest {
 			printEntries(entriesThree, "three");
 			performanceDb.put(entriesThree);
 
-			final List<Entry> actualEntriesOne = performanceDb.getAsList(tagsOne);
+			final List<Entry> actualEntriesOne = performanceDb.getAsList(Query.createQuery(tagsOne));
 			Assert.assertEquals(actualEntriesOne, entriesOne);
 
-			final List<Entry> actualEntriesTwo = performanceDb.getAsList(tagsTwo);
+			final List<Entry> actualEntriesTwo = performanceDb.getAsList(Query.createQuery(tagsTwo));
 			Assert.assertEquals(actualEntriesTwo, entriesTwo);
 
-			final List<Entry> actualEntriesThree = performanceDb.getAsList(tagsThree);
+			final List<Entry> actualEntriesThree = performanceDb.getAsList(Query.createQuery(tagsThree));
 			Assert.assertEquals(actualEntriesThree, entriesThree);
 
-			final List<Entry> actualEntriesAll = performanceDb.getAsList(tagsCommon);
+			final List<Entry> actualEntriesAll = performanceDb.getAsList(Query.createQuery(tagsCommon));
 			final List<Entry> expectedAll = CollectionUtils.collate(entriesOne,
 					CollectionUtils.collate(entriesTwo, entriesThree, Entry.BY_DATE), Entry.BY_DATE);