use TreeMap in PersistentMapDiskNode instead of list

This commit is contained in:
2018-11-24 15:57:05 +01:00
parent d67e452a91
commit 5404253bc6
3 changed files with 123 additions and 55 deletions

View File

@@ -0,0 +1,74 @@
package org.lucares.pdb.map;
import java.util.Arrays;
public final class ByteArrayKey implements Comparable<ByteArrayKey> {
private final byte[] bytes;
public ByteArrayKey(final byte[] bytes) {
this.bytes = bytes;
}
@Override
public int compareTo(final ByteArrayKey o) {
return compare(bytes, o.bytes);
}
public static int compare(final byte[] key, final byte[] otherKey) {
return Arrays.compare(key, otherKey);
}
public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) {
return compareKeyPrefix(key, keyPrefix) == 0;
}
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public static int compareKeyPrefix(final byte[] key, final byte[] prefix) {
int i = 0;
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
return key.length > prefix.length ? 0 : key.length - prefix.length;
}
public static boolean equal(final byte[] key, final byte[] otherKey) {
return compare(key, otherKey) == 0;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(bytes);
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ByteArrayKey other = (ByteArrayKey) obj;
if (!Arrays.equals(bytes, other.bytes))
return false;
return true;
}
}

View File

@@ -2,8 +2,8 @@ package org.lucares.pdb.map;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator; import java.util.Comparator;
import java.util.List;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Predicate; import java.util.function.Predicate;
@@ -47,6 +47,8 @@ class NodeEntry {
} }
} }
// TODO remove?
@Deprecated
public static final Comparator<NodeEntry> SORT_BY_KEY = (a, b) -> a.compare(b.getKey()); public static final Comparator<NodeEntry> SORT_BY_KEY = (a, b) -> a.compare(b.getKey());
private final ValueType type; private final ValueType type;
@@ -126,26 +128,18 @@ class NodeEntry {
return true; return true;
} }
public static int neededBytes(final List<NodeEntry> entries) { public static int neededBytes(final Collection<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum(); return entries.stream().mapToInt(NodeEntry::size).sum();
} }
public int compare(final byte[] otherKey) { public int compare(final byte[] otherKey) {
int i = 0; return ByteArrayKey.compare(key, otherKey);
while (i < key.length && i < otherKey.length) {
if (key[i] != otherKey[i]) {
return key[i] - otherKey[i];
}
i++;
}
return key.length - otherKey.length;
} }
public boolean isPrefix(final byte[] keyPrefix) { public boolean isPrefix(final byte[] keyPrefix) {
return compareKeyPrefix(keyPrefix) == 0; return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0;
} }
/** /**
@@ -159,16 +153,7 @@ class NodeEntry {
*/ */
public int compareKeyPrefix(final byte[] prefix) { public int compareKeyPrefix(final byte[] prefix) {
int i = 0; return ByteArrayKey.compareKeyPrefix(key, prefix);
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
return key.length > prefix.length ? 0 : key.length - prefix.length;
} }
public boolean equal(final byte[] otherKey) { public boolean equal(final byte[] otherKey) {

View File

@@ -2,8 +2,12 @@ package org.lucares.pdb.map;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.lucares.collections.LongList; import org.lucares.collections.LongList;
@@ -33,15 +37,25 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*/ */
public class PersistentMapDiskNode { public class PersistentMapDiskNode {
// TODO use map instead of list private final TreeMap<ByteArrayKey, NodeEntry> entries;
private final List<NodeEntry> entries;
private final long nodeOffset; private final long nodeOffset;
private final DiskBlock diskBlock; private final DiskBlock diskBlock;
public PersistentMapDiskNode(final long nodeOffset, final List<NodeEntry> entries, final DiskBlock diskBlock) { public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries,
final DiskBlock diskBlock) {
this.nodeOffset = nodeOffset; this.nodeOffset = nodeOffset;
this.diskBlock = diskBlock; this.diskBlock = diskBlock;
this.entries = new ArrayList<>(entries); this.entries = toMap(entries);
}
private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) {
final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>();
for (final NodeEntry nodeEntry : nodeEntries) {
result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry);
}
return result;
} }
public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) { public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) {
@@ -106,20 +120,14 @@ public class PersistentMapDiskNode {
public NodeEntry getNodeEntryTo(final byte[] key) { public NodeEntry getNodeEntryTo(final byte[] key) {
final NodeEntry result = null; final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key));
for (final NodeEntry entry : entries) { return ceilingEntry != null ? ceilingEntry.getValue() : null;
if (entry.compare(key) >= 0) {
return entry;
}
}
return result;
} }
public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) { public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) {
final List<NodeEntry> result = new ArrayList<>(); final List<NodeEntry> result = new ArrayList<>();
for (final NodeEntry nodeEntry : entries) { for (final NodeEntry nodeEntry : entries.values()) {
final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix); final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) { if (prefixCompareResult == 0) {
// add all entries where keyPrefix is a prefix of the key // add all entries where keyPrefix is a prefix of the key
@@ -148,11 +156,8 @@ public class PersistentMapDiskNode {
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) { public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
entries.removeIf(new NodeEntry.KeyMatches(key));
final NodeEntry entry = new NodeEntry(valueType, key, value); final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.add(entry); entries.put(new ByteArrayKey(key), entry);
Collections.sort(entries, NodeEntry.SORT_BY_KEY);
} }
public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) { public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) {
@@ -166,7 +171,7 @@ public class PersistentMapDiskNode {
} else { } else {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value); final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1); final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries); tmp.addAll(entries.values());
tmp.add(entry); tmp.add(entry);
// the +1 is for the null-byte terminator of the prefix // the +1 is for the null-byte terminator of the prefix
@@ -175,11 +180,11 @@ public class PersistentMapDiskNode {
} }
public void removeKey(final byte[] key) { public void removeKey(final byte[] key) {
entries.removeIf(entry -> entry.compare(key) == 0); entries.remove(new ByteArrayKey(key));
} }
public List<NodeEntry> getEntries() { public List<NodeEntry> getEntries() {
return new ArrayList<>(entries); return new ArrayList<>(entries.values());
} }
public void clear() { public void clear() {
@@ -189,20 +194,23 @@ public class PersistentMapDiskNode {
@Override @Override
public String toString() { public String toString() {
return "@" + nodeOffset + ": " return "@" + nodeOffset + ": "
+ String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList())); + String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList()));
} }
public NodeEntry getTopNodeEntry() { public NodeEntry getTopNodeEntry() {
return entries.get(entries.size() - 1); return entries.lastEntry().getValue();
} }
public PersistentMapDiskNode split(final long newBlockOffset) { public PersistentMapDiskNode split(final long newBlockOffset) {
final var leftEntries = new ArrayList<>(entries.subList(0, entries.size() / 2)); final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values());
final var rightEntries = new ArrayList<>(entries.subList(entries.size() / 2, entries.size()));
final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2));
final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
entries.clear(); entries.clear();
entries.addAll(rightEntries); entries.putAll(toMap(rightEntries));
return new PersistentMapDiskNode(newBlockOffset, leftEntries, null); return new PersistentMapDiskNode(newBlockOffset, leftEntries, null);
} }
@@ -215,27 +223,28 @@ public class PersistentMapDiskNode {
return usedBytes + NodeEntry.neededBytes(entries); return usedBytes + NodeEntry.neededBytes(entries);
} }
private static byte[] serialize(final List<NodeEntry> entries) { private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entries, buffer); final int usedBytes = serializePrefix(entriesAsCollection, buffer);
// the +1 is for the null-byte terminator of the prefix // the +1 is for the null-byte terminator of the prefix
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE, usedBytes + 1 + NodeEntry.neededBytes(entries), Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE,
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes."); "The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
serializeIntoFromTail(entries, buffer); serializeIntoFromTail(entriesAsCollection, buffer);
return buffer; return buffer;
} }
private static int serializePrefix(final List<NodeEntry> entries, final byte[] buffer) { private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) {
final LongList longs = serializeKeyLengths(entries); final LongList longs = serializeKeyLengths(entries);
final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0); final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0);
return usedBytes; return usedBytes;
} }
private static LongList serializeKeyLengths(final List<NodeEntry> entries) { private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) {
final var keyLengths = new LongList(); final var keyLengths = new LongList();
keyLengths.add(entries.size()); keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) { for (final NodeEntry nodeEntry : entries) {
@@ -246,7 +255,7 @@ public class PersistentMapDiskNode {
return keyLengths; return keyLengths;
} }
private static void serializeIntoFromTail(final List<NodeEntry> entries, final byte[] buffer) { private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) {
int offset = buffer.length; int offset = buffer.length;