diff --git a/block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java b/block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java index 53618cf..11efade 100644 --- a/block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java +++ b/block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java @@ -190,4 +190,10 @@ public class VariableByteEncoder { } return offset - offsetInBuffer; } + + public static int neededBytes(final long value) { + final byte[] buffer = SINGLE_VALUE_BUFFER.get(); + final int usedBytes = encodeInto(value, buffer, 0); + return usedBytes; + } } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java index addd517..ba2ad79 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java @@ -60,8 +60,11 @@ class NodeEntry { @Override public String toString() { + final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value)) + : new String(value, StandardCharsets.UTF_8); + return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value=" - + new String(value, StandardCharsets.UTF_8) + "]"; + + valueAsString + "]"; } @Override @@ -92,10 +95,6 @@ class NodeEntry { return true; } - public static int neededBytes(final List entries) { - return entries.stream().mapToInt(NodeEntry::size).sum(); - } - public static List deserialize(final byte[] buffer) { final List entries = new ArrayList<>(); final LongList keyLengths = VariableByteEncoder.decode(buffer); @@ -128,22 +127,39 @@ class NodeEntry { return entries; } - public static byte[] serialize(final List entries) { - final var keyLengths = new LongList(); + public static int neededBytes(final List entries) { + return entries.stream().mapToInt(NodeEntry::size).sum(); + } + public static int neededBytesTotal(final List entries) { + final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; + + final int usedBytes = serializeKeyLengths(entries, buffer); + + return usedBytes + NodeEntry.neededBytes(entries); + } + + public static byte[] serialize(final List entries) { + final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; + + final int usedBytes = serializeKeyLengths(entries, buffer); + + Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), ""); + + NodeEntry.serializeIntoFromTail(entries, buffer); + return buffer; + } + + private static int serializeKeyLengths(final List entries, final byte[] buffer) { + final var keyLengths = new LongList(); keyLengths.add(entries.size()); for (final NodeEntry nodeEntry : entries) { keyLengths.add(nodeEntry.getKey().length); keyLengths.add(nodeEntry.getValue().length); } - final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; final int usedBytes = VariableByteEncoder.encodeInto(keyLengths, buffer, 0); - - Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), ""); - - NodeEntry.serializeIntoFromTail(entries, buffer); - return buffer; + return usedBytes; } private static void serializeIntoFromTail(final List entries, final byte[] buffer) { diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java index a11ae0f..3f77442 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java @@ -3,6 +3,7 @@ package org.lucares.pdb.map; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.List; import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder; import org.lucares.pdb.diskstorage.DiskBlock; @@ -11,6 +12,10 @@ import org.lucares.utils.Preconditions; public class PersistentMap { + interface VisitorCallback { + void visit(NodeEntry nodeEntry, int depth); + } + private static final Charset UTF8 = StandardCharsets.UTF_8; static final int BLOCK_SIZE = 4096; private static final int ROOT_NODE_OFFEST = 4096; @@ -70,18 +75,33 @@ public class PersistentMap { final PersistentMapDiskNode node = getNode(nodeOffest); final var entry = node.getNodeEntryTo(key); - if (entry == null) { - node.addKeyValue(key, value); - writeNode(nodeOffest, node); - return null; - } else if (entry.isDataNode()) { - if (entry.equal(key)) { - return entry.getValue(); + if (entry == null || entry.isDataNode()) { + + final byte[] oldValue; + if (entry == null) { + oldValue = null; } else { - node.removeKey(key); + final boolean entryIsForKey = entry.equal(key); + + oldValue = entryIsForKey ? entry.getValue() : null; + + if (entryIsForKey) { + node.removeKey(key); + } + } + + if (node.canAdd(key, value)) { + // insert in existing node node.addKeyValue(key, value); writeNode(nodeOffest, node); - return null; + return oldValue; + } else { + // add new node + // 1. split current node into A and B + splitNode(nodeOffest, node); + + // 2. insert the value + return insert(nodeOffest, key, value); } } else { final long childNodeOffset = toNodeOffset(entry); @@ -89,6 +109,31 @@ public class PersistentMap { } } + private void splitNode(final long nodeOffest, final PersistentMapDiskNode node) throws IOException { + + final long newLeftBlockOffset = diskStore.allocateBlock(BLOCK_SIZE); + final long newRightBlockOffset = diskStore.allocateBlock(BLOCK_SIZE); + + final List entries = node.getEntries(); + final var leftEntries = entries.subList(0, entries.size() / 2); + final var rightEntries = entries.subList(entries.size() / 2, entries.size()); + + final PersistentMapDiskNode leftNode = new PersistentMapDiskNode(leftEntries); + final PersistentMapDiskNode rightNode = new PersistentMapDiskNode(rightEntries); + + node.clear(); + + final NodeEntry lastLeftEntry = leftEntries.get(leftEntries.size() - 1); + final NodeEntry lastRightEntry = rightEntries.get(rightEntries.size() - 1); + + node.addKeyNodePointer(lastLeftEntry.getKey(), newLeftBlockOffset); + node.addKeyNodePointer(lastRightEntry.getKey(), newRightBlockOffset); + + writeNode(newLeftBlockOffset, leftNode); + writeNode(newRightBlockOffset, rightNode); + writeNode(nodeOffest, node); + } + private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) throws IOException { final PersistentMapDiskNode node = getNode(nodeOffest); @@ -127,4 +172,28 @@ public class PersistentMap { diskBlock.writeAsync(); diskBlock.force(); } + + public void print() throws IOException { + + visitPreOrder((nodeEntry, depth) -> System.out.println(" ".repeat(depth) + nodeEntry)); + + } + + public void visitPreOrder(final VisitorCallback visitor) throws IOException { + visitPreOrderRecursively(ROOT_NODE_OFFEST, visitor, 0); + } + + private void visitPreOrderRecursively(final long nodeOffset, final VisitorCallback visitor, final int depth) + throws IOException { + final PersistentMapDiskNode node = getNode(nodeOffset); + + for (final NodeEntry child : node.getEntries()) { + + visitor.visit(child, depth); + if (child.isInnerNode()) { + final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue()); + visitPreOrderRecursively(childNodeOffset, visitor, depth + 1); + } + } + } } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java index 81f013b..562ffcf 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java @@ -1,8 +1,11 @@ package org.lucares.pdb.map; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; +import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder; import org.lucares.pdb.map.NodeEntry.ValueType; /** @@ -38,25 +41,56 @@ public class PersistentMapDiskNode { public NodeEntry getNodeEntryTo(final byte[] key) { - NodeEntry result = null; + final NodeEntry result = null; for (final NodeEntry entry : entries) { - if (entry.compare(key) <= 0) { - result = entry; + // if (entry.compare(key) <= 0) { + if (entry.compare(key) >= 0) { + return entry; } else { - break; + // break; } } return result; } public void addKeyValue(final byte[] key, final byte[] value) { - final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value); + addNode(ValueType.VALUE_INLINE, key, value); + } + + public void addKeyNodePointer(final byte[] key, final long nodePointer) { + final byte[] value = VariableByteEncoder.encode(nodePointer); + addNode(ValueType.NODE_POINTER, key, value); + } + + public void addNode(final ValueType valueType, final byte[] key, final byte[] value) { + final NodeEntry entry = new NodeEntry(valueType, key, value); entries.add(entry); Collections.sort(entries, NodeEntry.SORT_BY_KEY); } + public boolean canAdd(final byte[] key, final byte[] value) { + final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value); + final List tmp = new ArrayList<>(entries.size() + 1); + tmp.addAll(entries); + tmp.add(entry); + return NodeEntry.neededBytesTotal(tmp) <= PersistentMap.BLOCK_SIZE; + } + public void removeKey(final byte[] key) { entries.removeIf(entry -> entry.compare(key) == 0); } + public List getEntries() { + return new ArrayList<>(entries); + } + + public void clear() { + entries.clear(); + } + + @Override + public String toString() { + return String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList())); + } + } diff --git a/block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java b/block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java index 5437ada..a95543b 100644 --- a/block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java +++ b/block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java @@ -81,4 +81,29 @@ public class VariableByteEncoderTest { final LongList decodedValues = VariableByteEncoder.decode(buffer); Assert.assertEquals(decodedValues, originalValues); } + + @DataProvider + public Object[][] providerNededBytes() { + return new Object[][] { // + { 0, 1 }, // + { -10, 1 }, // + { 10, 1 }, // + { -63, 1 }, // + { 63, 1 }, // + { -64, 2 }, // + { 64, 2 }, // + { -8191, 2 }, // + { 8191, 2 }, // + { -8192, 3 }, // + { 8192, 3 }, // + }; + } + + @Test(dataProvider = "providerNededBytes") + public void testNeededBytes(final long value, final int expectedNeededBytes) { + + final int neededBytes = VariableByteEncoder.neededBytes(value); + final byte[] encoded = VariableByteEncoder.encode(value); + Assert.assertEquals(encoded.length, neededBytes); + } } diff --git a/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java b/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java index 7883ac0..80c8193 100644 --- a/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java +++ b/block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java @@ -3,6 +3,8 @@ package org.lucares.pdb.map; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; +import java.util.UUID; import org.lucares.pdb.diskstorage.DiskStorage; import org.lucares.utils.file.FileUtils; @@ -38,9 +40,51 @@ public class PersistentMapTest { Assert.assertNull(map.getAsString(key)); Assert.assertNull(map.put(key, value)); - final String actualValue = map.getAsString(key); - Assert.assertEquals(actualValue, value); + Assert.assertEquals(map.getAsString(key), value); } } + + public void testManyValues() throws Exception { + final Path file = dataDirectory.resolve("map.db"); + final var insertedValues = new HashMap(); + + try (final DiskStorage ds = new DiskStorage(file)) { + final PersistentMap map = new PersistentMap(ds); + + for (int i = 0; i < 100; i++) { + + final String key = UUID.randomUUID().toString() + "__" + i; + final String value = "long value to waste some bytes " + i; + Assert.assertNull(map.getAsString(key)); + + Assert.assertNull(map.put(key, value)); + + insertedValues.put(key, value); + + for (final var entry : insertedValues.entrySet()) { + final String actualValue = map.getAsString(entry.getKey()); + Assert.assertEquals(actualValue, entry.getValue(), + "value for key " + entry.getKey() + " in the " + i + "th iteration"); + } + } + } + + try (final DiskStorage ds = new DiskStorage(file)) { + final PersistentMap map = new PersistentMap(ds); + map.visitPreOrder((nodeEntry, depth) -> { + if (nodeEntry.isInnerNode()) { + System.out.println(" ".repeat(depth) + nodeEntry); + } + }); + + for (final var entry : insertedValues.entrySet()) { + final String actualValue = map.getAsString(entry.getKey()); + Assert.assertEquals(actualValue, entry.getValue(), + "value for key " + entry.getKey() + " after all iterations"); + } + + } + } + }