diff --git a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java index 3816957..81becb5 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java @@ -137,6 +137,34 @@ class NodeEntry { return key.length - otherKey.length; } + public boolean isPrefix(final byte[] keyPrefix) { + + return compareKeyPrefix(keyPrefix) == 0; + } + + /** + * Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the + * key. {@link #compare(byte[])} return values >0 in that case, because key + * is longer than the prefix. + * + * @param prefix the prefix + * @return 0 if {@code prefix} is a prefix of the key otherwise the value is + * defined by {@link #compare(byte[])} + */ + public int compareKeyPrefix(final byte[] prefix) { + + int i = 0; + while (i < key.length && i < prefix.length) { + if (key[i] != prefix[i]) { + return key[i] - prefix[i]; + } + i++; + } + + return key.length > prefix.length ? 0 : key.length - prefix.length; + + } + public boolean equal(final byte[] otherKey) { return compare(otherKey) == 0; } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java index 46d18e2..7117ed6 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java @@ -4,6 +4,9 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Stack; import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder; @@ -28,6 +31,10 @@ public class PersistentMap { void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth); } + interface Visitor { + void visit(K key, V value); + } + public interface EncoderDecoder { public byte[] encode(O object); @@ -109,6 +116,12 @@ public class PersistentMap { } } + public void putAllValues(final Map map) throws IOException { + for (final Entry e : map.entrySet()) { + putValue(e.getKey(), e.getValue()); + } + } + public V putValue(final K key, final V value) throws IOException { final byte[] encodedKey = keyEncoder.encode(key); final byte[] encodedValue = valueEncoder.encode(value); @@ -305,6 +318,44 @@ public class PersistentMap { } } + enum VisitByPrefixMode { + FIND, ITERATE + } + + public void visitValues(final K keyPrefix, final Visitor visitor) throws IOException { + final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix); + + final long rootNodeOffset = readNodeOffsetOfRootNode(); + iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor); + } + + private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor visitor) + throws IOException { + final PersistentMapDiskNode node = getNode(nodeOffest); + + // list of children that might contain a key with the keyPrefix + final List nodesForPrefix = node.getNodesByPrefix(keyPrefix); + + for (final NodeEntry entry : nodesForPrefix) { + + if (entry.isDataNode()) { + final int prefixCompareResult = entry.compareKeyPrefix(keyPrefix); + if (prefixCompareResult == 0) { + + final K key = keyEncoder.decode(entry.getKey()); + final V value = valueEncoder.decode(entry.getValue()); + visitor.visit(key, value); + // System.out.println("--> " + key + "=" + value); + } else if (prefixCompareResult > 0) { + break; + } + } else { + final long childNodeOffset = toNodeOffset(entry); + iterateNodeEntryByPrefix(childNodeOffset, keyPrefix, visitor); + } + } + } + private long readNodeOffsetOfRootNode() throws IOException { final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize()); @@ -316,4 +367,5 @@ public class PersistentMap { diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode); diskBlock.force(); } + } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java index 460b7b7..597a3b4 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java @@ -102,13 +102,32 @@ public class PersistentMapDiskNode { if (entry.compare(key) >= 0) { return entry; - } else { - // break; } } return result; } + public List getNodesByPrefix(final byte[] keyPrefix) { + final List result = new ArrayList<>(); + + for (final NodeEntry nodeEntry : entries) { + final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix); + if (prefixCompareResult == 0) { + // add all entries where keyPrefix is a prefix of the key + result.add(nodeEntry); + } else if (prefixCompareResult > 0) { + // Only add the first entry where the keyPrefix is smaller (as defined by + // compareKeyPrefix) than the key. + // These are entries that might contain key with the keyPrefix. But only the + // first of those can really have such keys. + result.add(nodeEntry); + break; + } + } + + return result; + } + public void addKeyValue(final byte[] key, final byte[] value) { addNode(ValueType.VALUE_INLINE, key, value); } diff --git a/block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java b/block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java index f66005d..cf26d35 100644 --- a/block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java +++ b/block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java @@ -1,7 +1,37 @@ package org.lucares.pdb.map; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.lucares.pdb.map.NodeEntry.ValueType; +import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @Test public class NodeEntryTest { + @DataProvider + public Object[][] providerPrefixCompare() { + final List result = new ArrayList<>(); + + result.add(new Object[] { "ab", "abc", -1 }); + result.add(new Object[] { "abb", "abc", -1 }); + result.add(new Object[] { "abc", "abc", 0 }); + result.add(new Object[] { "abcd", "abc", 0 }); + result.add(new Object[] { "abd", "abc", 1 }); + result.add(new Object[] { "abz", "abc", 23 }); + + return result.toArray(Object[][]::new); + } + + @Test(dataProvider = "providerPrefixCompare") + public void testPrefixCompare(final String key, final String prefix, final int expected) { + + final NodeEntry nodeEntry = new NodeEntry(ValueType.NODE_POINTER, key.getBytes(StandardCharsets.UTF_8), + new byte[0]); + + final int actual = nodeEntry.compareKeyPrefix(prefix.getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(actual, expected, key + " ? " + prefix); + } } diff --git a/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java b/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java index d3f4f0f..6f65d7e 100644 --- a/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java +++ b/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java @@ -6,7 +6,9 @@ import java.nio.file.Path; import java.security.SecureRandom; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; +import java.util.Map; import java.util.Objects; import java.util.Queue; import java.util.Random; @@ -14,6 +16,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import org.lucares.pdb.diskstorage.DiskStorage; +import org.lucares.pdb.map.PersistentMap.Visitor; import org.lucares.utils.file.FileUtils; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -238,4 +241,42 @@ public class PersistentMapTest { } } + @Test + public void testFindAllByPrefix() throws Exception { + final Path file = dataDirectory.resolve("map.db"); + + final Map expectedBar = new HashMap<>(); + for (int i = 0; i < 100; i++) { + // the value is a little bit longer to make sure that the values don't fit into + // a single leaf node + expectedBar.put("bar:" + i, "bar:" + i + "__##################################"); + } + + final Map input = new HashMap<>(); + input.putAll(expectedBar); + for (int i = 0; i < 500; i++) { + input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + + try (final DiskStorage ds = new DiskStorage(file)) { + final PersistentMap map = new PersistentMap<>(ds, PersistentMap.STRING_CODER, + PersistentMap.STRING_CODER); + + map.putAllValues(input); + } + + try (final DiskStorage ds = new DiskStorage(file)) { + final PersistentMap map = new PersistentMap<>(ds, PersistentMap.STRING_CODER, + PersistentMap.STRING_CODER); + + { + final LinkedHashMap actualBar = new LinkedHashMap<>(); + final Visitor visitor = (key, value) -> actualBar.put(key, value); + map.visitValues("bar:", visitor); + + Assert.assertEquals(actualBar, expectedBar); + } + } + } + }