rewrite query completion

The old implementation searched for all possible values and then
executed each query to see what matches.
The new implementation uses several indices to find only
the matching values.

block-storage/src/main/java/org/lucares/pdb/map/PersistentMapOfListsOfLongs.java

@@ -0,0 +1,107 @@
+package org.lucares.pdb.map;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.NoSuchElementException;
+import java.util.stream.Stream;
+
+import org.lucares.collections.LongList;
+import org.lucares.pdb.blockstorage.BSFile;
+import org.lucares.pdb.diskstorage.DiskStorage;
+import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
+import org.lucares.utils.Preconditions;
+import org.lucares.utils.cache.HotEntryCache;
+import org.lucares.utils.cache.HotEntryCache.Event;
+import org.lucares.utils.cache.HotEntryCache.EventListener;
+import org.lucares.utils.cache.HotEntryCache.EventType;
+
+/**
+ * Combines {@link PersistentMap} and {@link BSFile} to represent a map where
+ * the values are lists of longs.
+ */
+public class PersistentMapOfListsOfLongs<K> implements AutoCloseable {
+
+	private static final class RemovalListener<KEY> implements EventListener<KEY, BSFile> {
+		@Override
+		public void onEvent(final Event<KEY, BSFile> event) {
+			event.getValue().close();
+		}
+	}
+
+	private final PersistentMap<K, Long> map;
+	private final Path mapPath;
+	private final DiskStorage diskStore;
+	private final Path diskStorePath;
+
+	private final HotEntryCache<K, BSFile> writerCache;
+
+	/**
+	 * Creates a new map that stores indexed streams/lists of longs.
+	 * <p>
+	 * This class creates two files on disk. One for the index and one for the lists
+	 * of longs.
+	 *
+	 * @param path       the folder where to store the map
+	 * @param filePrefix prefix of the files
+	 * @param keyEncoder {@link EncoderDecoder} for the key
+	 * @throws IOException
+	 */
+	public PersistentMapOfListsOfLongs(final Path path, final String filePrefix, final EncoderDecoder<K> keyEncoder)
+			throws IOException {
+		Preconditions.checkTrue(Files.isDirectory(path), "must be a directory {0}", path);
+		mapPath = path.resolve(filePrefix + "_index.bs");
+		diskStorePath = path.resolve(filePrefix + "_data.bs");
+		map = new PersistentMap<>(mapPath, keyEncoder, PersistentMap.LONG_CODER);
+		diskStore = new DiskStorage(diskStorePath);
+
+		writerCache = new HotEntryCache<>(Duration.ofMinutes(10), filePrefix + "Cache");
+		writerCache.addListener(new RemovalListener<K>(), EventType.EVICTED, EventType.REMOVED);
+	}
+
+	public synchronized void appendLong(final K key, final long value) throws IOException {
+
+		BSFile cachedWriter = writerCache.get(key);
+		if (cachedWriter == null) {
+			final Long bsFileBlockNumber = map.getValue(key);
+
+			if (bsFileBlockNumber == null) {
+				cachedWriter = BSFile.newFile(diskStore);
+				map.putValue(key, cachedWriter.getRootBlockOffset());
+			} else {
+				cachedWriter = BSFile.existingFile(bsFileBlockNumber, diskStore);
+			}
+			writerCache.put(key, cachedWriter);
+		}
+		cachedWriter.append(value);
+	}
+
+	public synchronized boolean hasKey(final K key) throws IOException {
+		return map.getValue(key) != null;
+	}
+
+	public synchronized Stream<LongList> getLongs(final K key) throws IOException {
+		final Long bsFileBlockNumber = map.getValue(key);
+		if (bsFileBlockNumber == null) {
+			throw new NoSuchElementException("the map at '" + mapPath + "' does not contain the key '" + key + "'");
+		}
+
+		final BSFile bsFile = BSFile.existingFile(bsFileBlockNumber, diskStore);
+
+		return bsFile.streamOfLongLists();
+	}
+
+	@Override
+	public void close() throws IOException {
+		try {
+			try {
+				writerCache.forEach(bsFile -> bsFile.close());
+			} finally {
+				map.close();
+			}
+		} finally {
+			diskStore.close();
+		}
+	}
+}

block-storage/src/test/java/org/lucares/pdb/map/PersistentMapOfListsOfLongsTest.java

@@ -0,0 +1,62 @@
+package org.lucares.pdb.map;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.lucares.collections.LongList;
+import org.lucares.utils.file.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test
+public class PersistentMapOfListsOfLongsTest {
+
+	private Path dataDirectory;
+
+	@BeforeMethod
+	public void beforeMethod() throws IOException {
+		dataDirectory = Files.createTempDirectory("pdb");
+	}
+
+	@AfterMethod
+	public void afterMethod() throws IOException {
+		FileUtils.delete(dataDirectory);
+	}
+
+	public void test() throws IOException {
+
+		final String mapFilePrefix = "test";
+		final String keyA = "a";
+		final String keyB = "b";
+
+		final int size = 10;
+		final LongList a = LongList.range(0, size);
+		a.shuffle();
+		final LongList b = LongList.range(0, size);
+		b.shuffle();
+
+		try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
+				PersistentMap.STRING_CODER)) {
+
+			for (int i = 0; i < size; i++) {
+				map.appendLong(keyA, a.get(i));
+				map.appendLong(keyB, b.get(i));
+			}
+		}
+
+		try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
+				PersistentMap.STRING_CODER)) {
+
+			final LongList actualA = new LongList();
+			map.getLongs(keyA).forEachOrdered(actualA::addAll);
+			Assert.assertEquals(actualA, a);
+
+			final LongList actualB = new LongList();
+			map.getLongs(keyB).forEachOrdered(actualB::addAll);
+			Assert.assertEquals(actualB, b);
+		}
+	}
+}