From 5404253bc6d91b6cfa866e6000b74529996c7070 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 24 Nov 2018 15:57:05 +0100 Subject: [PATCH] use TreeMap in PersistentMapDiskNode instead of list --- .../org/lucares/pdb/map/ByteArrayKey.java | 74 ++++++++++++++++++ .../java/org/lucares/pdb/map/NodeEntry.java | 29 ++----- .../pdb/map/PersistentMapDiskNode.java | 75 +++++++++++-------- 3 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java diff --git a/block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java b/block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java new file mode 100644 index 0000000..c15b220 --- /dev/null +++ b/block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java @@ -0,0 +1,74 @@ +package org.lucares.pdb.map; + +import java.util.Arrays; + +public final class ByteArrayKey implements Comparable { + private final byte[] bytes; + + public ByteArrayKey(final byte[] bytes) { + this.bytes = bytes; + } + + @Override + public int compareTo(final ByteArrayKey o) { + return compare(bytes, o.bytes); + } + + public static int compare(final byte[] key, final byte[] otherKey) { + return Arrays.compare(key, otherKey); + } + + public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) { + + return compareKeyPrefix(key, keyPrefix) == 0; + } + + /** + * Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the + * key. {@link #compare(byte[])} return values >0 in that case, because key + * is longer than the prefix. + * + * @param prefix the prefix + * @return 0 if {@code prefix} is a prefix of the key otherwise the value is + * defined by {@link #compare(byte[])} + */ + public static int compareKeyPrefix(final byte[] key, final byte[] prefix) { + int i = 0; + while (i < key.length && i < prefix.length) { + if (key[i] != prefix[i]) { + return key[i] - prefix[i]; + } + i++; + } + + return key.length > prefix.length ? 0 : key.length - prefix.length; + + } + + public static boolean equal(final byte[] key, final byte[] otherKey) { + return compare(key, otherKey) == 0; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(bytes); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final ByteArrayKey other = (ByteArrayKey) obj; + if (!Arrays.equals(bytes, other.bytes)) + return false; + return true; + } + +} diff --git a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java index cf1fbe9..0ce98b2 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java @@ -2,8 +2,8 @@ package org.lucares.pdb.map; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; -import java.util.List; import java.util.function.Function; import java.util.function.Predicate; @@ -47,6 +47,8 @@ class NodeEntry { } } + // TODO remove? + @Deprecated public static final Comparator SORT_BY_KEY = (a, b) -> a.compare(b.getKey()); private final ValueType type; @@ -126,26 +128,18 @@ class NodeEntry { return true; } - public static int neededBytes(final List entries) { + public static int neededBytes(final Collection entries) { return entries.stream().mapToInt(NodeEntry::size).sum(); } public int compare(final byte[] otherKey) { - int i = 0; - while (i < key.length && i < otherKey.length) { - if (key[i] != otherKey[i]) { - return key[i] - otherKey[i]; - } - i++; - } - - return key.length - otherKey.length; + return ByteArrayKey.compare(key, otherKey); } public boolean isPrefix(final byte[] keyPrefix) { - return compareKeyPrefix(keyPrefix) == 0; + return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0; } /** @@ -159,16 +153,7 @@ class NodeEntry { */ public int compareKeyPrefix(final byte[] prefix) { - int i = 0; - while (i < key.length && i < prefix.length) { - if (key[i] != prefix[i]) { - return key[i] - prefix[i]; - } - i++; - } - - return key.length > prefix.length ? 0 : key.length - prefix.length; - + return ByteArrayKey.compareKeyPrefix(key, prefix); } public boolean equal(final byte[] otherKey) { diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java index 3838438..3253ddb 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java @@ -2,8 +2,12 @@ package org.lucares.pdb.map; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; import java.util.stream.Collectors; import org.lucares.collections.LongList; @@ -33,15 +37,25 @@ import org.lucares.utils.byteencoder.VariableByteEncoder; */ public class PersistentMapDiskNode { - // TODO use map instead of list - private final List entries; + private final TreeMap entries; private final long nodeOffset; private final DiskBlock diskBlock; - public PersistentMapDiskNode(final long nodeOffset, final List entries, final DiskBlock diskBlock) { + public PersistentMapDiskNode(final long nodeOffset, final Collection entries, + final DiskBlock diskBlock) { this.nodeOffset = nodeOffset; this.diskBlock = diskBlock; - this.entries = new ArrayList<>(entries); + this.entries = toMap(entries); + } + + private static TreeMap toMap(final Collection nodeEntries) { + final TreeMap result = new TreeMap<>(); + + for (final NodeEntry nodeEntry : nodeEntries) { + result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry); + } + + return result; } public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) { @@ -106,20 +120,14 @@ public class PersistentMapDiskNode { public NodeEntry getNodeEntryTo(final byte[] key) { - final NodeEntry result = null; - for (final NodeEntry entry : entries) { - - if (entry.compare(key) >= 0) { - return entry; - } - } - return result; + final Entry ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key)); + return ceilingEntry != null ? ceilingEntry.getValue() : null; } public List getNodesByPrefix(final byte[] keyPrefix) { final List result = new ArrayList<>(); - for (final NodeEntry nodeEntry : entries) { + for (final NodeEntry nodeEntry : entries.values()) { final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix); if (prefixCompareResult == 0) { // add all entries where keyPrefix is a prefix of the key @@ -148,11 +156,8 @@ public class PersistentMapDiskNode { public void addNode(final ValueType valueType, final byte[] key, final byte[] value) { - entries.removeIf(new NodeEntry.KeyMatches(key)); - final NodeEntry entry = new NodeEntry(valueType, key, value); - entries.add(entry); - Collections.sort(entries, NodeEntry.SORT_BY_KEY); + entries.put(new ByteArrayKey(key), entry); } public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) { @@ -166,7 +171,7 @@ public class PersistentMapDiskNode { } else { final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value); final List tmp = new ArrayList<>(entries.size() + 1); - tmp.addAll(entries); + tmp.addAll(entries.values()); tmp.add(entry); // the +1 is for the null-byte terminator of the prefix @@ -175,11 +180,11 @@ public class PersistentMapDiskNode { } public void removeKey(final byte[] key) { - entries.removeIf(entry -> entry.compare(key) == 0); + entries.remove(new ByteArrayKey(key)); } public List getEntries() { - return new ArrayList<>(entries); + return new ArrayList<>(entries.values()); } public void clear() { @@ -189,20 +194,23 @@ public class PersistentMapDiskNode { @Override public String toString() { return "@" + nodeOffset + ": " - + String.join("\n", entries.stream().map(NodeEntry::toString).collect(Collectors.toList())); + + String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList())); } public NodeEntry getTopNodeEntry() { - return entries.get(entries.size() - 1); + return entries.lastEntry().getValue(); } public PersistentMapDiskNode split(final long newBlockOffset) { - final var leftEntries = new ArrayList<>(entries.subList(0, entries.size() / 2)); - final var rightEntries = new ArrayList<>(entries.subList(entries.size() / 2, entries.size())); + final List entriesAsCollection = new ArrayList<>(entries.values()); + + final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2)); + final var rightEntries = new ArrayList<>( + entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size())); entries.clear(); - entries.addAll(rightEntries); + entries.putAll(toMap(rightEntries)); return new PersistentMapDiskNode(newBlockOffset, leftEntries, null); } @@ -215,27 +223,28 @@ public class PersistentMapDiskNode { return usedBytes + NodeEntry.neededBytes(entries); } - private static byte[] serialize(final List entries) { + private static byte[] serialize(final Map entries) { final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; - - final int usedBytes = serializePrefix(entries, buffer); + final Collection entriesAsCollection = entries.values(); + final int usedBytes = serializePrefix(entriesAsCollection, buffer); // the +1 is for the null-byte terminator of the prefix - Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE, usedBytes + 1 + NodeEntry.neededBytes(entries), + Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE, + usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection), "The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes."); - serializeIntoFromTail(entries, buffer); + serializeIntoFromTail(entriesAsCollection, buffer); return buffer; } - private static int serializePrefix(final List entries, final byte[] buffer) { + private static int serializePrefix(final Collection entries, final byte[] buffer) { final LongList longs = serializeKeyLengths(entries); final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0); return usedBytes; } - private static LongList serializeKeyLengths(final List entries) { + private static LongList serializeKeyLengths(final Collection entries) { final var keyLengths = new LongList(); keyLengths.add(entries.size()); for (final NodeEntry nodeEntry : entries) { @@ -246,7 +255,7 @@ public class PersistentMapDiskNode { return keyLengths; } - private static void serializeIntoFromTail(final List entries, final byte[] buffer) { + private static void serializeIntoFromTail(final Collection entries, final byte[] buffer) { int offset = buffer.length;