move creation of PdbWriter to the DataStore

data-store/src/main/java/org/lucares/pdb/datastore/InvalidValueException.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 public class InvalidValueException extends IllegalArgumentException {
 

data-store/src/main/java/org/lucares/pdb/datastore/PdbFile.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 import java.io.IOException;
 import java.util.List;
@@ -11,7 +11,7 @@ import org.lucares.pdb.api.Tags;
 import org.lucares.pdb.blockstorage.BSFile;
 import org.lucares.pdb.diskstorage.DiskStorage;
 
-class PdbFile {
+public class PdbFile {
 
 	private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
 

data-store/src/main/java/org/lucares/pdb/datastore/PdbWriter.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 import java.io.Flushable;
 import java.io.IOException;
@@ -13,7 +13,7 @@ import org.slf4j.LoggerFactory;
 /**
  *
  */
-class PdbWriter implements AutoCloseable, Flushable {
+public class PdbWriter implements AutoCloseable, Flushable {
 
 	private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
 
@@ -22,7 +22,7 @@ class PdbWriter implements AutoCloseable, Flushable {
 
 	private final BSFile bsFile;
 
-	PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) throws IOException {
+	public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) throws IOException {
 		this.pdbFile = pdbFile;
 
 		bsFile = BSFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);

data-store/src/main/java/org/lucares/pdb/datastore/ReadException.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 import java.io.IOException;
 

data-store/src/main/java/org/lucares/pdb/datastore/ReadRuntimeException.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 public class ReadRuntimeException extends RuntimeException {
 

data-store/src/main/java/org/lucares/pdb/datastore/WriteException.java

@@ -1,4 +1,4 @@
-package org.lucares.performance.db;
+package org.lucares.pdb.datastore;
 
 public class WriteException extends RuntimeException {
 

data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java

@@ -9,6 +9,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -21,7 +22,11 @@ import org.lucares.pdb.api.Tag;
 import org.lucares.pdb.api.Tags;
 import org.lucares.pdb.blockstorage.BSFile;
 import org.lucares.pdb.datastore.Doc;
+import org.lucares.pdb.datastore.PdbFile;
+import org.lucares.pdb.datastore.PdbWriter;
 import org.lucares.pdb.datastore.Proposal;
+import org.lucares.pdb.datastore.ReadException;
+import org.lucares.pdb.datastore.WriteException;
 import org.lucares.pdb.datastore.lang.Expression;
 import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
 import org.lucares.pdb.datastore.lang.NewProposerParser;
@@ -42,6 +47,8 @@ public class DataStore implements AutoCloseable {
 			.getLogger("org.lucares.metrics.dataStore.executeQuery");
 	private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
 			.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
+	private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
+			.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
 	private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
 
 	public static final char LISTING_FILE_SEPARATOR = ',';
@@ -330,15 +337,14 @@ public class DataStore implements AutoCloseable {
 		return result;
 	}
 
-	public List<Doc> getByTags(final Tags tags) {
+	public Optional<Doc> getByTags(final Tags tags) {
 		try {
 			final Long docId = tagsToDocId.getValue(tags);
-			final List<Doc> result = new ArrayList<>(0);
 			if (docId != null) {
 				final Doc doc = getDocByDocId(docId);
-				result.add(doc);
+				return Optional.of(doc);
 			}
-			return result;
+			return Optional.empty();
 		} catch (final IOException e) {
 			throw new RuntimeIOException(e);
 		}
@@ -375,4 +381,44 @@ public class DataStore implements AutoCloseable {
 		return diskStorage;
 	}
 
+	public PdbWriter getWriter(final Tags tags) {
+		final Optional<Doc> docsForTags = getByTags(tags);
+		PdbWriter writer;
+		if (docsForTags.isPresent()) {
+			try {
+				final Doc doc = docsForTags.get();
+				final PdbFile pdbFile = new PdbFile(doc.getRootBlockNumber(), tags);
+				writer = new PdbWriter(pdbFile, getDiskStorage());
+			} catch (final IOException e) {
+				throw new ReadException(e);
+			}
+		} else {
+			writer = newPdbWriter(tags);
+		}
+		return writer;
+	}
+
+	private PdbWriter newPdbWriter(final Tags tags) {
+		final long start = System.nanoTime();
+		try {
+			final PdbFile pdbFile = createNewPdbFile(tags);
+			final PdbWriter result = new PdbWriter(pdbFile, getDiskStorage());
+
+			METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
+					(System.nanoTime() - start) / 1_000_000.0, tags);
+			return result;
+		} catch (final IOException e) {
+			throw new WriteException(e);
+		}
+
+	}
+
+	private PdbFile createNewPdbFile(final Tags tags) throws IOException {
+
+		final long rootBlockNumber = createNewFile(tags);
+
+		final PdbFile result = new PdbFile(rootBlockNumber, tags);
+		return result;
+	}
+
 }

data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java

@@ -14,6 +14,7 @@ import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -110,8 +111,8 @@ public class DataStoreTest {
 		tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(pigeonJennifer));
 		tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(flamingoJennifer));
 
-		final List<Doc> docsFlamingoJennifer = dataStore.getByTags(flamingoJennifer);
+		final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(flamingoJennifer);
-		Assert.assertEquals(docsFlamingoJennifer.size(), 1, "doc for docsFlamingoJennifer");
+		Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
 	}
 
 	public void testBlockAlignment() throws IOException {

performanceDb/src/main/java/org/lucares/performance/db/Group.java

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
 import java.util.List;
 
 import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.datastore.PdbFile;
 
 class Group {
 	private final Tags tags;

performanceDb/src/main/java/org/lucares/performance/db/Grouping.java

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.datastore.PdbFile;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

performanceDb/src/main/java/org/lucares/performance/db/PdbExport.java

@@ -23,6 +23,7 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.lucares.collections.LongList;
 import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.datastore.PdbFile;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -82,7 +83,7 @@ public class PdbExport {
 
 			for (final PdbFile pdbFile : pdbFiles) {
 
-				if (writer == null || Files.size(exportFile) > GB) {
+				if (writer == null || Files.size(exportFile) > 4 * GB) {
 					if (writer != null) {
 						writer.flush();
 						writer.close();
@@ -100,7 +101,6 @@ public class PdbExport {
 				final Tags tags = pdbFile.getTags();
 				final long tagsId = addNewTagsToDictionary(writer, tags, tagsIdCounter);
 
-				
 				final Iterator<LongList> it = timeValueStream.iterator();
 				while (it.hasNext()) {
 					final LongList entry = it.next();
@@ -123,10 +123,11 @@ public class PdbExport {
 						count++;
 						final long chunk = 10_000_000;
 						if (count % chunk == 0) {
-							long end = System.currentTimeMillis();
+							final long end = System.currentTimeMillis();
-							long duration = end-begin;
+							final long duration = end - begin;
-							long entriesPerSecond = (long)((double)chunk / (duration / 1000.0));
+							final long entriesPerSecond = (long) (chunk / (duration / 1000.0));
-							LOGGER.info("progress: {} - {} entries/s + duration {}" , String.format("%,d",count), String.format("%,d",entriesPerSecond), duration);
+							LOGGER.info("progress: {} - {} entries/s + duration {}", String.format("%,d", count),
+									String.format("%,d", entriesPerSecond), duration);
 							begin = System.currentTimeMillis();
 						}
 					}

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -17,7 +17,11 @@ import org.lucares.pdb.api.Entry;
 import org.lucares.pdb.api.GroupResult;
 import org.lucares.pdb.api.Result;
 import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.datastore.InvalidValueException;
+import org.lucares.pdb.datastore.PdbFile;
+import org.lucares.pdb.datastore.PdbWriter;
 import org.lucares.pdb.datastore.Proposal;
+import org.lucares.pdb.datastore.WriteException;
 import org.lucares.pdb.datastore.internal.DataStore;
 import org.lucares.pdb.datastore.lang.SyntaxException;
 import org.lucares.pdb.diskstorage.DiskStorage;

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -1,6 +1,5 @@
 package org.lucares.performance.db;
 
-import java.io.IOException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
@@ -8,6 +7,10 @@ import java.util.function.Consumer;
 
 import org.lucares.pdb.api.Tags;
 import org.lucares.pdb.datastore.Doc;
+import org.lucares.pdb.datastore.PdbFile;
+import org.lucares.pdb.datastore.PdbWriter;
+import org.lucares.pdb.datastore.ReadException;
+import org.lucares.pdb.datastore.WriteException;
 import org.lucares.pdb.datastore.internal.DataStore;
 import org.lucares.utils.cache.HotEntryCache;
 import org.lucares.utils.cache.HotEntryCache.Event;
@@ -19,8 +22,6 @@ import org.slf4j.LoggerFactory;
 public class TagsToFile implements AutoCloseable {
 
 	private static final Logger LOGGER = LoggerFactory.getLogger(TagsToFile.class);
-	private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
-			.getLogger("org.lucares.metrics.ingestion.tagsToFile.newPdbWriter");
 
 	private static final class CacheKey implements Comparable<CacheKey> {
 		private final Tags tags;
@@ -77,7 +78,6 @@ public class TagsToFile implements AutoCloseable {
 
 		writerCache = new HotEntryCache<>(Duration.ofSeconds(10), "writerCache");
 		writerCache.addListener(new RemovalListener(), EventType.EVICTED, EventType.REMOVED);
-
 	}
 
 	public List<PdbFile> getFilesForQuery(final String query) {
@@ -115,18 +115,7 @@ public class TagsToFile implements AutoCloseable {
 				if (writer == null) {
 
 					LOGGER.trace("getByTags({})", tags);
-					final List<Doc> docsForTags = dataStore.getByTags(tags);
+					writer = dataStore.getWriter(tags);
-					if (docsForTags.size() > 0) {
-						try {
-							final Doc doc = docsForTags.get(0);
-							final PdbFile pdbFile = new PdbFile(doc.getRootBlockNumber(), tags);
-							writer = new PdbWriter(pdbFile, dataStore.getDiskStorage());
-						} catch (final IOException e) {
-							throw new ReadException(e);
-						}
-					} else {
-						writer = newPdbWriter(tags);
-					}
 					writerCache.put(cacheKey, writer);
 				}
 			}
@@ -134,29 +123,6 @@ public class TagsToFile implements AutoCloseable {
 		return writer;
 	}
 
-	private PdbWriter newPdbWriter(final Tags tags) {
-		final long start = System.nanoTime();
-		try {
-			final PdbFile pdbFile = createNewPdbFile(tags);
-			final PdbWriter result = new PdbWriter(pdbFile, dataStore.getDiskStorage());
-
-			METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
-					(System.nanoTime() - start) / 1_000_000.0, tags);
-			return result;
-		} catch (final IOException e) {
-			throw new WriteException(e);
-		}
-
-	}
-
-	private PdbFile createNewPdbFile(final Tags tags) throws IOException {
-
-		final long rootBlockNumber = dataStore.createNewFile(tags);
-
-		final PdbFile result = new PdbFile(rootBlockNumber, tags);
-		return result;
-	}
-
 	private void forEachWriter(final Consumer<PdbWriter> consumer) {
 		writerCache.forEach(writer -> {
 			try {

performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java

@@ -8,6 +8,7 @@ import java.time.ZoneOffset;
 
 import org.lucares.pdb.api.Entry;
 import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.datastore.PdbWriter;
 import org.lucares.pdb.datastore.internal.DataStore;
 import org.testng.Assert;
 import org.testng.annotations.AfterMethod;