andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use TreeMap in PersistentMapDiskNode instead of list

Browse Source
This commit is contained in:
Andreas Huber
2018-11-24 15:57:05 +01:00
parent d67e452a91
commit 5404253bc6
3 changed files with 123 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java Normal file
View File

@@ -0,0 +1,74 @@
package org.lucares.pdb.map;
import java.util.Arrays;
public final class ByteArrayKey implements Comparable<ByteArrayKey> {
private final byte[] bytes;
public ByteArrayKey(final byte[] bytes) {
this.bytes = bytes;
}
@Override
public int compareTo(final ByteArrayKey o) {
return compare(bytes, o.bytes);
}
public static int compare(final byte[] key, final byte[] otherKey) {
return Arrays.compare(key, otherKey);
}
public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) {
return compareKeyPrefix(key, keyPrefix) == 0;
}
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public static int compareKeyPrefix(final byte[] key, final byte[] prefix) {
int i = 0;
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
return key.length > prefix.length ? 0 : key.length - prefix.length;
}
public static boolean equal(final byte[] key, final byte[] otherKey) {
return compare(key, otherKey) == 0;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(bytes);
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ByteArrayKey other = (ByteArrayKey) obj;
if (!Arrays.equals(bytes, other.bytes))
return false;
return true;
}
}

29
block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java
View File

@@ -2,8 +2,8 @@ package org.lucares.pdb.map;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.function.Function;
import java.util.function.Predicate;
@@ -47,6 +47,8 @@ class NodeEntry {
}
}
// TODO remove?
@Deprecated
public static final Comparator<NodeEntry> SORT_BY_KEY = (a, b) -> a.compare(b.getKey());
private final ValueType type;
@@ -126,26 +128,18 @@ class NodeEntry {
return true;
}
public static int neededBytes(final List<NodeEntry> entries) {
public static int neededBytes(final Collection<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum();
}
public int compare(final byte[] otherKey) {
int i = 0;
while (i < key.length && i < otherKey.length) {
if (key[i] != otherKey[i]) {
return key[i] - otherKey[i];
}
i++;
}
return key.length - otherKey.length;
return ByteArrayKey.compare(key, otherKey);
}
public boolean isPrefix(final byte[] keyPrefix) {
return compareKeyPrefix(keyPrefix) == 0;
return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0;
}
/**
@@ -159,16 +153,7 @@ class NodeEntry {
*/
public int compareKeyPrefix(final byte[] prefix) {
int i = 0;
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
return key.length > prefix.length ? 0 : key.length - prefix.length;
return ByteArrayKey.compareKeyPrefix(key, prefix);
}
public boolean equal(final byte[] otherKey) {

75
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java
View File

@@ -2,8 +2,12 @@ package org.lucares.pdb.map;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.lucares.collections.LongList;
@@ -33,15 +37,25 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*/
public class PersistentMapDiskNode {
// TODO use map instead of list
private final List<NodeEntry> entries;
private final TreeMap<ByteArrayKey, NodeEntry> entries;
private final long nodeOffset;
private final DiskBlock diskBlock;
public PersistentMapDiskNode(final long nodeOffset, final List<NodeEntry> entries, final DiskBlock diskBlock) {
public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries,
final DiskBlock diskBlock) {
this.nodeOffset = nodeOffset;
this.diskBlock = diskBlock;
this.entries = new ArrayList<>(entries);
this.entries = toMap(entries);
}
private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) {
final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>();
for (final NodeEntry nodeEntry : nodeEntries) {
result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry);
}
return result;
}
public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) {
@@ -106,20 +120,14 @@ public class PersistentMapDiskNode {
public NodeEntry getNodeEntryTo(final byte[] key) {
final NodeEntry result = null;
for (final NodeEntry entry : entries) {
if (entry.compare(key) >= 0) {
return entry;
}
}
return result;
final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key));
return ceilingEntry != null ? ceilingEntry.getValue() : null;
}
public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) {
final List<NodeEntry> result = new ArrayList<>();
for (final NodeEntry nodeEntry : entries) {
for (final NodeEntry nodeEntry : entries.values()) {
final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) {
// add all entries where keyPrefix is a prefix of the key
@@ -148,11 +156,8 @@ public class PersistentMapDiskNode {
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
entries.removeIf(new NodeEntry.KeyMatches(key));
final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.add(entry);
Collections.sort(entries, NodeEntry.SORT_BY_KEY);
entries.put(new ByteArrayKey(key), entry);
}
public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) {
@@ -166,7 +171,7 @@ public class PersistentMapDiskNode {
} else {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries);
tmp.addAll(entries.values());
tmp.add(entry);
// the +1 is for the null-byte terminator of the prefix
@@ -175,11 +180,11 @@ public class PersistentMapDiskNode {
}
public void removeKey(final byte[] key) {
entries.removeIf(entry -> entry.compare(key) == 0);
entries.remove(new ByteArrayKey(key));
}
public List<NodeEntry> getEntries() {
return new ArrayList<>(entries);
return new ArrayList<>(entries.values());
}
public void clear() {
@@ -189,20 +194,23 @@ public class PersistentMapDiskNode {
@Override
public String toString() {
return "@" + nodeOffset + ": "
+ String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList()));
+ String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList()));
}
public NodeEntry getTopNodeEntry() {
return entries.get(entries.size() - 1);
return entries.lastEntry().getValue();
}
public PersistentMapDiskNode split(final long newBlockOffset) {
final var leftEntries = new ArrayList<>(entries.subList(0, entries.size() / 2));
final var rightEntries = new ArrayList<>(entries.subList(entries.size() / 2, entries.size()));
final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values());
final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2));
final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
entries.clear();
entries.addAll(rightEntries);
entries.putAll(toMap(rightEntries));
return new PersistentMapDiskNode(newBlockOffset, leftEntries, null);
}
@@ -215,27 +223,28 @@ public class PersistentMapDiskNode {
return usedBytes + NodeEntry.neededBytes(entries);
}
private static byte[] serialize(final List<NodeEntry> entries) {
private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final int usedBytes = serializePrefix(entries, buffer);
final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entriesAsCollection, buffer);
// the +1 is for the null-byte terminator of the prefix
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE, usedBytes + 1 + NodeEntry.neededBytes(entries),
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE,
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
serializeIntoFromTail(entries, buffer);
serializeIntoFromTail(entriesAsCollection, buffer);
return buffer;
}
private static int serializePrefix(final List<NodeEntry> entries, final byte[] buffer) {
private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) {
final LongList longs = serializeKeyLengths(entries);
final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0);
return usedBytes;
}
private static LongList serializeKeyLengths(final List<NodeEntry> entries) {
private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) {
final var keyLengths = new LongList();
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
@@ -246,7 +255,7 @@ public class PersistentMapDiskNode {
return keyLengths;
}
private static void serializeIntoFromTail(final List<NodeEntry> entries, final byte[] buffer) {
private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) {
int offset = buffer.length;