andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

PersistentMap can store data in multiple nodes

Browse Source
This commit is contained in:
Andreas Huber
2018-10-26 18:35:32 +02:00
parent bb4514c940
commit 8bb98deb1e
6 changed files with 223 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
block-storage/src/main/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoder.java
View File

@@ -190,4 +190,10 @@ public class VariableByteEncoder {
}
return offset - offsetInBuffer;
}
public static int neededBytes(final long value) {
final byte[] buffer = SINGLE_VALUE_BUFFER.get();
final int usedBytes = encodeInto(value, buffer, 0);
return usedBytes;
}
}

42
block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java
View File

@@ -60,8 +60,11 @@ class NodeEntry {
@Override
public String toString() {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: new String(value, StandardCharsets.UTF_8);
return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value="
+ new String(value, StandardCharsets.UTF_8) + "]";
+ valueAsString + "]";
}
@Override
@@ -92,10 +95,6 @@ class NodeEntry {
return true;
}
public static int neededBytes(final List<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum();
}
public static List<NodeEntry> deserialize(final byte[] buffer) {
final List<NodeEntry> entries = new ArrayList<>();
final LongList keyLengths = VariableByteEncoder.decode(buffer);
@@ -128,22 +127,39 @@ class NodeEntry {
return entries;
}
public static byte[] serialize(final List<NodeEntry> entries) {
final var keyLengths = new LongList();
public static int neededBytes(final List<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum();
}
public static int neededBytesTotal(final List<NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final int usedBytes = serializeKeyLengths(entries, buffer);
return usedBytes + NodeEntry.neededBytes(entries);
}
public static byte[] serialize(final List<NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final int usedBytes = serializeKeyLengths(entries, buffer);
Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), "");
NodeEntry.serializeIntoFromTail(entries, buffer);
return buffer;
}
private static int serializeKeyLengths(final List<NodeEntry> entries, final byte[] buffer) {
final var keyLengths = new LongList();
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
keyLengths.add(nodeEntry.getKey().length);
keyLengths.add(nodeEntry.getValue().length);
}
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final int usedBytes = VariableByteEncoder.encodeInto(keyLengths, buffer, 0);
Preconditions.checkGreater(PersistentMap.BLOCK_SIZE, usedBytes + NodeEntry.neededBytes(entries), "");
NodeEntry.serializeIntoFromTail(entries, buffer);
return buffer;
return usedBytes;
}
private static void serializeIntoFromTail(final List<NodeEntry> entries, final byte[] buffer) {

83
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -3,6 +3,7 @@ package org.lucares.pdb.map;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.List;
import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
import org.lucares.pdb.diskstorage.DiskBlock;
@@ -11,6 +12,10 @@ import org.lucares.utils.Preconditions;
public class PersistentMap {
interface VisitorCallback {
void visit(NodeEntry nodeEntry, int depth);
}
private static final Charset UTF8 = StandardCharsets.UTF_8;
static final int BLOCK_SIZE = 4096;
private static final int ROOT_NODE_OFFEST = 4096;
@@ -70,18 +75,33 @@ public class PersistentMap {
final PersistentMapDiskNode node = getNode(nodeOffest);
final var entry = node.getNodeEntryTo(key);
if (entry == null || entry.isDataNode()) {
final byte[] oldValue;
if (entry == null) {
node.addKeyValue(key, value);
writeNode(nodeOffest, node);
return null;
} else if (entry.isDataNode()) {
if (entry.equal(key)) {
return entry.getValue();
oldValue = null;
} else {
final boolean entryIsForKey = entry.equal(key);
oldValue = entryIsForKey ? entry.getValue() : null;
if (entryIsForKey) {
node.removeKey(key);
}
}
if (node.canAdd(key, value)) {
// insert in existing node
node.addKeyValue(key, value);
writeNode(nodeOffest, node);
return null;
return oldValue;
} else {
// add new node
// 1. split current node into A and B
splitNode(nodeOffest, node);
// 2. insert the value
return insert(nodeOffest, key, value);
}
} else {
final long childNodeOffset = toNodeOffset(entry);
@@ -89,6 +109,31 @@ public class PersistentMap {
}
}
private void splitNode(final long nodeOffest, final PersistentMapDiskNode node) throws IOException {
final long newLeftBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final long newRightBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final List<NodeEntry> entries = node.getEntries();
final var leftEntries = entries.subList(0, entries.size() / 2);
final var rightEntries = entries.subList(entries.size() / 2, entries.size());
final PersistentMapDiskNode leftNode = new PersistentMapDiskNode(leftEntries);
final PersistentMapDiskNode rightNode = new PersistentMapDiskNode(rightEntries);
node.clear();
final NodeEntry lastLeftEntry = leftEntries.get(leftEntries.size() - 1);
final NodeEntry lastRightEntry = rightEntries.get(rightEntries.size() - 1);
node.addKeyNodePointer(lastLeftEntry.getKey(), newLeftBlockOffset);
node.addKeyNodePointer(lastRightEntry.getKey(), newRightBlockOffset);
writeNode(newLeftBlockOffset, leftNode);
writeNode(newRightBlockOffset, rightNode);
writeNode(nodeOffest, node);
}
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) throws IOException {
final PersistentMapDiskNode node = getNode(nodeOffest);
@@ -127,4 +172,28 @@ public class PersistentMap {
diskBlock.writeAsync();
diskBlock.force();
}
public void print() throws IOException {
visitPreOrder((nodeEntry, depth) -> System.out.println(" ".repeat(depth) + nodeEntry));
}
public void visitPreOrder(final VisitorCallback visitor) throws IOException {
visitPreOrderRecursively(ROOT_NODE_OFFEST, visitor, 0);
}
private void visitPreOrderRecursively(final long nodeOffset, final VisitorCallback visitor, final int depth)
throws IOException {
final PersistentMapDiskNode node = getNode(nodeOffset);
for (final NodeEntry child : node.getEntries()) {
visitor.visit(child, depth);
if (child.isInnerNode()) {
final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue());
visitPreOrderRecursively(childNodeOffset, visitor, depth + 1);
}
}
}
}

44
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java
View File

@@ -1,8 +1,11 @@
package org.lucares.pdb.map;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
import org.lucares.pdb.map.NodeEntry.ValueType;
/**
@@ -38,25 +41,56 @@ public class PersistentMapDiskNode {
public NodeEntry getNodeEntryTo(final byte[] key) {
NodeEntry result = null;
final NodeEntry result = null;
for (final NodeEntry entry : entries) {
if (entry.compare(key) <= 0) {
result = entry;
// if (entry.compare(key) <= 0) {
if (entry.compare(key) >= 0) {
return entry;
} else {
break;
// break;
}
}
return result;
}
public void addKeyValue(final byte[] key, final byte[] value) {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
addNode(ValueType.VALUE_INLINE, key, value);
}
public void addKeyNodePointer(final byte[] key, final long nodePointer) {
final byte[] value = VariableByteEncoder.encode(nodePointer);
addNode(ValueType.NODE_POINTER, key, value);
}
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.add(entry);
Collections.sort(entries, NodeEntry.SORT_BY_KEY);
}
public boolean canAdd(final byte[] key, final byte[] value) {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries);
tmp.add(entry);
return NodeEntry.neededBytesTotal(tmp) <= PersistentMap.BLOCK_SIZE;
}
public void removeKey(final byte[] key) {
entries.removeIf(entry -> entry.compare(key) == 0);
}
public List<NodeEntry> getEntries() {
return new ArrayList<>(entries);
}
public void clear() {
entries.clear();
}
@Override
public String toString() {
return String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList()));
}
}

25
block-storage/src/test/java/org/lucares/pdb/blockstorage/intsequence/VariableByteEncoderTest.java
View File

@@ -81,4 +81,29 @@ public class VariableByteEncoderTest {
final LongList decodedValues = VariableByteEncoder.decode(buffer);
Assert.assertEquals(decodedValues, originalValues);
}
@DataProvider
public Object[][] providerNededBytes() {
return new Object[][] { //
{ 0, 1 }, //
{ -10, 1 }, //
{ 10, 1 }, //
{ -63, 1 }, //
{ 63, 1 }, //
{ -64, 2 }, //
{ 64, 2 }, //
{ -8191, 2 }, //
{ 8191, 2 }, //
{ -8192, 3 }, //
{ 8192, 3 }, //
};
}
@Test(dataProvider = "providerNededBytes")
public void testNeededBytes(final long value, final int expectedNeededBytes) {
final int neededBytes = VariableByteEncoder.neededBytes(value);
final byte[] encoded = VariableByteEncoder.encode(value);
Assert.assertEquals(encoded.length, neededBytes);
}
}

48
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -3,6 +3,8 @@ package org.lucares.pdb.map;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.UUID;
import org.lucares.pdb.diskstorage.DiskStorage;
import org.lucares.utils.file.FileUtils;
@@ -38,9 +40,51 @@ public class PersistentMapTest {
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.put(key, value));
final String actualValue = map.getAsString(key);
Assert.assertEquals(actualValue, value);
Assert.assertEquals(map.getAsString(key), value);
}
}
public void testManyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
for (int i = 0; i < 100; i++) {
final String key = UUID.randomUUID().toString() + "__" + i;
final String value = "long value to waste some bytes " + i;
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.put(key, value));
insertedValues.put(key, value);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
map.visitPreOrder((nodeEntry, depth) -> {
if (nodeEntry.isInnerNode()) {
System.out.println(" ".repeat(depth) + nodeEntry);
}
});
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
}