leverage the cached pdbwriters

this increased performance from 500 entries per second to 4000.

performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java

@@ -107,7 +107,6 @@ class PdbWriter implements AutoCloseable, Flushable {
 	}
 
 	public void write(final Entry entry) throws WriteException {
-		System.out.println(entry);
 		final long epochMilli = entry.getEpochMilli();
 		final long value = entry.getValue();
 		write(epochMilli, value);

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -57,8 +57,9 @@ public class PerformanceDb implements AutoCloseable {
 
 	public void put(final BlockingIterator<Entry> entries) throws WriteException {
 
+		final int blocksize = 10000;
 		long count = 0;
-		double durationInManager = 0;
+		final double durationInManager = 0;
 
 		try {
 			long start = System.nanoTime();
@@ -74,22 +75,25 @@ public class PerformanceDb implements AutoCloseable {
 				final Tags tags = entry.getTags();
 				final OffsetDateTime date = entry.getDate();
 
+				final long s = System.nanoTime();
 				final PdbWriter writer = tagsToFile.getWriter(date, tags);
+				final long e = System.nanoTime();
+				Stats.duration += (e - s) / 1_000_000.0;
 
 				writer.write(entry);
 				count++;
 
-				if (count == 100000) {
+				if (count % blocksize == 0) {
 					final long end = System.nanoTime();
 					final double duration = (end - start) / 1_000_000.0;
-					LOGGER.info("inserting the last " + count + " took " + duration + " ms; " + durationInManager
-							+ "ms in entries.next ");
+					LOGGER.info("inserting the last " + blocksize + " took " + duration + " ms; " + Stats.duration
+							+ "ms of " + Stats.count + " operations. total entries: " + count);
 
-					System.out.println(entry);
+					// System.out.println(entry);
 
 					start = System.nanoTime();
-					durationInManager = 0.0;
-					count = 0;
+					Stats.duration = 0.0;
+					Stats.count = 0;
 				}
 			}
 
@@ -100,6 +104,7 @@ public class PerformanceDb implements AutoCloseable {
 			LOGGER.info("Thread was interrupted. Aborting exectution.");
 		} finally {
 			tagsToFile.flush();
+			LOGGER.info("flushed all files.");
 		}
 	}
 

performanceDb/src/main/java/org/lucares/performance/db/Stats.java

@@ -0,0 +1,13 @@
+package org.lucares.performance.db;
+
+public class Stats {
+	public static double duration = 0.0;
+
+	public static long count = 0;
+
+	{
+		final long s = System.nanoTime();
+		final long e = System.nanoTime();
+		Stats.duration += (e - s) / 1_000_000.0;
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -157,27 +157,39 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
 	}
 
 	public PdbWriter getWriter(final OffsetDateTime date, final Tags tags) throws ReadException, WriteException {
-
-		final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
-
-		assertAllFilesHaveSameFolder(pdbFiles);
-
+		final PdbWriter result;
 		final WriterCache writersForTags = getOrInit(tags);
 
-		pdbFiles.removeIf(f -> !f.exists());
-		final List<Optional<PdbWriter>> optionalWriters = map(pdbFiles, writersForTags::writer);
-		final List<Optional<PdbWriter>> existingWriters = filter(optionalWriters, Optional::isPresent);
-		final List<PdbWriter> writers = map(existingWriters, Optional::get);
+		final Optional<PdbWriter> optionalWriter = chooseBestMatchingWriter(writersForTags.getWriters(), date);
+
+		if (optionalWriter.isPresent()) {
+			result = optionalWriter.get();
+		} else {
+			Stats.count++;
+			final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
+
+			assertAllFilesHaveSameFolder(pdbFiles);
+
+			pdbFiles.removeIf(f -> !f.exists());
+			final List<Optional<PdbWriter>> optionalWriters = map(pdbFiles, writersForTags::writer);
+			final List<Optional<PdbWriter>> existingWriters = filter(optionalWriters, Optional::isPresent);
+			final List<PdbWriter> writers = map(existingWriters, Optional::get);
+
+			final Optional<PdbWriter> optionalFirst = chooseBestMatchingWriter(writers, date);
+
+			result = optionalFirst.orElseGet(() -> newPdbWriter(tags));
+		}
+		return result;
+	}
+
+	private Optional<PdbWriter> chooseBestMatchingWriter(final List<PdbWriter> writers, final OffsetDateTime date) {
 		final List<PdbWriter> candidateWriters = filter(writers, writer -> {
 			final OffsetDateTime offsetTime = writer.getDateOffset();
 			return !date.isBefore(offsetTime);
 		});
 		final List<PdbWriter> sortedCanditateWriters = sorted(candidateWriters, PdbWriterByTimeAsc.INSTANCE.reversed());
 		final Optional<PdbWriter> optionalFirst = findFirst(sortedCanditateWriters);
-
-		final PdbWriter result = optionalFirst.orElseGet(() -> newPdbWriter(tags));
-
-		return result;
+		return optionalFirst;
 	}
 
 	private WriterCache getOrInit(final Tags tags) {