PersistentMap can store data in multiple nodes

block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java

@@ -190,4 +190,10 @@ public class VariableByteEncoder {
 		}
 		return offset - offsetInBuffer;
 	}
+
+	public static int neededBytes(final long value) {
+		final byte[] buffer = SINGLE_VALUE_BUFFER.get();
+		final int usedBytes = encodeInto(value, buffer, 0);
+		return usedBytes;
+	}
 }

block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java

@@ -60,8 +60,11 @@ class NodeEntry {
 
 	@Override
 	public String toString() {
+		final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
+				: new String(value, StandardCharsets.UTF_8);
+
 		return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value="
-				+ new String(value, StandardCharsets.UTF_8) + "]";
+				+ valueAsString + "]";
 	}
 
 	@Override
@@ -92,10 +95,6 @@ class NodeEntry {
 		return true;
 	}
 
-	public static int neededBytes(final List<NodeEntry> entries) {
-		return entries.stream().mapToInt(NodeEntry::size).sum();
-	}
-
 	public static List<NodeEntry> deserialize(final byte[] buffer) {
 		final List<NodeEntry> entries = new ArrayList<>();
 		final LongList keyLengths = VariableByteEncoder.decode(buffer);
@@ -128,22 +127,39 @@ class NodeEntry {
 		return entries;
 	}
 
-	public static byte[] serialize(final List<NodeEntry> entries) {
+	public static int neededBytes(final List<NodeEntry> entries) {
-		final var keyLengths = new LongList();
+		return entries.stream().mapToInt(NodeEntry::size).sum();
+	}
 
+	public static int neededBytesTotal(final List<NodeEntry> entries) {
+		final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
+
+		final int usedBytes = serializeKeyLengths(entries, buffer);
+
+		return usedBytes + NodeEntry.neededBytes(entries);
+	}
+
+	public static byte[] serialize(final List<NodeEntry> entries) {
+		final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
+
+		final int usedBytes = serializeKeyLengths(entries, buffer);
+
+		Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), "");
+
+		NodeEntry.serializeIntoFromTail(entries, buffer);
+		return buffer;
+	}
+
+	private static int serializeKeyLengths(final List<NodeEntry> entries, final byte[] buffer) {
+		final var keyLengths = new LongList();
 		keyLengths.add(entries.size());
 		for (final NodeEntry nodeEntry : entries) {
 			keyLengths.add(nodeEntry.getKey().length);
 			keyLengths.add(nodeEntry.getValue().length);
 		}
 
-		final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
 		final int usedBytes = VariableByteEncoder.encodeInto(keyLengths, buffer, 0);
-
+		return usedBytes;
-		Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), "");
-
-		NodeEntry.serializeIntoFromTail(entries, buffer);
-		return buffer;
 	}
 
 	private static void serializeIntoFromTail(final List<NodeEntry> entries, final byte[] buffer) {

block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java

@@ -3,6 +3,7 @@ package org.lucares.pdb.map;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.List;
 
 import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
 import org.lucares.pdb.diskstorage.DiskBlock;
@@ -11,6 +12,10 @@ import org.lucares.utils.Preconditions;
 
 public class PersistentMap {
 
+	interface VisitorCallback {
+		void visit(NodeEntry nodeEntry, int depth);
+	}
+
 	private static final Charset UTF8 = StandardCharsets.UTF_8;
 	static final int BLOCK_SIZE = 4096;
 	private static final int ROOT_NODE_OFFEST = 4096;
@@ -70,18 +75,33 @@ public class PersistentMap {
 		final PersistentMapDiskNode node = getNode(nodeOffest);
 
 		final var entry = node.getNodeEntryTo(key);
+		if (entry == null || entry.isDataNode()) {
+
+			final byte[] oldValue;
 			if (entry == null) {
-			node.addKeyValue(key, value);
+				oldValue = null;
-			writeNode(nodeOffest, node);
-			return null;
-		} else if (entry.isDataNode()) {
-			if (entry.equal(key)) {
-				return entry.getValue();
 			} else {
+				final boolean entryIsForKey = entry.equal(key);
+
+				oldValue = entryIsForKey ? entry.getValue() : null;
+
+				if (entryIsForKey) {
 					node.removeKey(key);
+				}
+			}
+
+			if (node.canAdd(key, value)) {
+				// insert in existing node
 				node.addKeyValue(key, value);
 				writeNode(nodeOffest, node);
-				return null;
+				return oldValue;
+			} else {
+				// add new node
+				// 1. split current node into A and B
+				splitNode(nodeOffest, node);
+
+				// 2. insert the value
+				return insert(nodeOffest, key, value);
 			}
 		} else {
 			final long childNodeOffset = toNodeOffset(entry);
@@ -89,6 +109,31 @@ public class PersistentMap {
 		}
 	}
 
+	private void splitNode(final long nodeOffest, final PersistentMapDiskNode node) throws IOException {
+
+		final long newLeftBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
+		final long newRightBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
+
+		final List<NodeEntry> entries = node.getEntries();
+		final var leftEntries = entries.subList(0, entries.size() / 2);
+		final var rightEntries = entries.subList(entries.size() / 2, entries.size());
+
+		final PersistentMapDiskNode leftNode = new PersistentMapDiskNode(leftEntries);
+		final PersistentMapDiskNode rightNode = new PersistentMapDiskNode(rightEntries);
+
+		node.clear();
+
+		final NodeEntry lastLeftEntry = leftEntries.get(leftEntries.size() - 1);
+		final NodeEntry lastRightEntry = rightEntries.get(rightEntries.size() - 1);
+
+		node.addKeyNodePointer(lastLeftEntry.getKey(), newLeftBlockOffset);
+		node.addKeyNodePointer(lastRightEntry.getKey(), newRightBlockOffset);
+
+		writeNode(newLeftBlockOffset, leftNode);
+		writeNode(newRightBlockOffset, rightNode);
+		writeNode(nodeOffest, node);
+	}
+
 	private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) throws IOException {
 		final PersistentMapDiskNode node = getNode(nodeOffest);
 
@@ -127,4 +172,28 @@ public class PersistentMap {
 		diskBlock.writeAsync();
 		diskBlock.force();
 	}
+
+	public void print() throws IOException {
+
+		visitPreOrder((nodeEntry, depth) -> System.out.println("    ".repeat(depth) + nodeEntry));
+
+	}
+
+	public void visitPreOrder(final VisitorCallback visitor) throws IOException {
+		visitPreOrderRecursively(ROOT_NODE_OFFEST, visitor, 0);
+	}
+
+	private void visitPreOrderRecursively(final long nodeOffset, final VisitorCallback visitor, final int depth)
+			throws IOException {
+		final PersistentMapDiskNode node = getNode(nodeOffset);
+
+		for (final NodeEntry child : node.getEntries()) {
+
+			visitor.visit(child, depth);
+			if (child.isInnerNode()) {
+				final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue());
+				visitPreOrderRecursively(childNodeOffset, visitor, depth + 1);
+			}
+		}
+	}
 }

block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java

@@ -1,8 +1,11 @@
 package org.lucares.pdb.map;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
+import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
 import org.lucares.pdb.map.NodeEntry.ValueType;
 
 /**
@@ -38,25 +41,56 @@ public class PersistentMapDiskNode {
 
 	public NodeEntry getNodeEntryTo(final byte[] key) {
 
-		NodeEntry result = null;
+		final NodeEntry result = null;
 		for (final NodeEntry entry : entries) {
-			if (entry.compare(key) <= 0) {
+			// if (entry.compare(key) <= 0) {
-				result = entry;
+			if (entry.compare(key) >= 0) {
+				return entry;
 			} else {
-				break;
+				// break;
 			}
 		}
 		return result;
 	}
 
 	public void addKeyValue(final byte[] key, final byte[] value) {
-		final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
+		addNode(ValueType.VALUE_INLINE, key, value);
+	}
+
+	public void addKeyNodePointer(final byte[] key, final long nodePointer) {
+		final byte[] value = VariableByteEncoder.encode(nodePointer);
+		addNode(ValueType.NODE_POINTER, key, value);
+	}
+
+	public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
+		final NodeEntry entry = new NodeEntry(valueType, key, value);
 		entries.add(entry);
 		Collections.sort(entries, NodeEntry.SORT_BY_KEY);
 	}
 
+	public boolean canAdd(final byte[] key, final byte[] value) {
+		final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
+		final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
+		tmp.addAll(entries);
+		tmp.add(entry);
+		return NodeEntry.neededBytesTotal(tmp) <= PersistentMap.BLOCK_SIZE;
+	}
+
 	public void removeKey(final byte[] key) {
 		entries.removeIf(entry -> entry.compare(key) == 0);
 	}
 
+	public List<NodeEntry> getEntries() {
+		return new ArrayList<>(entries);
+	}
+
+	public void clear() {
+		entries.clear();
+	}
+
+	@Override
+	public String toString() {
+		return String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList()));
+	}
+
 }

block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java

@@ -81,4 +81,29 @@ public class VariableByteEncoderTest {
 		final LongList decodedValues = VariableByteEncoder.decode(buffer);
 		Assert.assertEquals(decodedValues, originalValues);
 	}
+
+	@DataProvider
+	public Object[][] providerNededBytes() {
+		return new Object[][] { //
+				{ 0, 1 }, //
+				{ -10, 1 }, //
+				{ 10, 1 }, //
+				{ -63, 1 }, //
+				{ 63, 1 }, //
+				{ -64, 2 }, //
+				{ 64, 2 }, //
+				{ -8191, 2 }, //
+				{ 8191, 2 }, //
+				{ -8192, 3 }, //
+				{ 8192, 3 }, //
+		};
+	}
+
+	@Test(dataProvider = "providerNededBytes")
+	public void testNeededBytes(final long value, final int expectedNeededBytes) {
+
+		final int neededBytes = VariableByteEncoder.neededBytes(value);
+		final byte[] encoded = VariableByteEncoder.encode(value);
+		Assert.assertEquals(encoded.length, neededBytes);
+	}
 }

block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java

@@ -3,6 +3,8 @@ package org.lucares.pdb.map;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.UUID;
 
 import org.lucares.pdb.diskstorage.DiskStorage;
 import org.lucares.utils.file.FileUtils;
@@ -38,9 +40,51 @@ public class PersistentMapTest {
 			Assert.assertNull(map.getAsString(key));
 
 			Assert.assertNull(map.put(key, value));
-			final String actualValue = map.getAsString(key);
 
-			Assert.assertEquals(actualValue, value);
+			Assert.assertEquals(map.getAsString(key), value);
+		}
+	}
+
+	public void testManyValues() throws Exception {
+		final Path file = dataDirectory.resolve("map.db");
+		final var insertedValues = new HashMap<String, String>();
+
+		try (final DiskStorage ds = new DiskStorage(file)) {
+			final PersistentMap map = new PersistentMap(ds);
+
+			for (int i = 0; i < 100; i++) {
+
+				final String key = UUID.randomUUID().toString() + "__" + i;
+				final String value = "long value to waste some bytes " + i;
+				Assert.assertNull(map.getAsString(key));
+
+				Assert.assertNull(map.put(key, value));
+
+				insertedValues.put(key, value);
+
+				for (final var entry : insertedValues.entrySet()) {
+					final String actualValue = map.getAsString(entry.getKey());
+					Assert.assertEquals(actualValue, entry.getValue(),
+							"value for key " + entry.getKey() + " in the " + i + "th iteration");
 				}
 			}
 		}
+
+		try (final DiskStorage ds = new DiskStorage(file)) {
+			final PersistentMap map = new PersistentMap(ds);
+			map.visitPreOrder((nodeEntry, depth) -> {
+				if (nodeEntry.isInnerNode()) {
+					System.out.println("    ".repeat(depth) + nodeEntry);
+				}
+			});
+
+			for (final var entry : insertedValues.entrySet()) {
+				final String actualValue = map.getAsString(entry.getKey());
+				Assert.assertEquals(actualValue, entry.getValue(),
+						"value for key " + entry.getKey() + " after all iterations");
+			}
+
+		}
+	}
+
+}