andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add generics to PersistencMap

Browse Source
This commit is contained in:
Andreas Huber
2018-11-04 10:42:05 +01:00
parent f2d5c27668
commit 008f0db377
3 changed files with 105 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -2,10 +2,9 @@ package org.lucares.pdb.map;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Stack;
import java.util.function.Function;
import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
import org.lucares.pdb.diskstorage.DiskBlock;
@@ -14,16 +13,16 @@ import org.lucares.utils.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PersistentMap {
public class PersistentMap<K, V> {
private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class);
// the maximum key
private static final byte[] MAX_KEY = new byte[] { Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE,
Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE,
Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE,
Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE,
Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE };
private static final byte[] MAX_KEY;
static {
MAX_KEY = new byte[20];
Arrays.fill(MAX_KEY, Byte.MAX_VALUE);
}
interface VisitorCallback {
void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth);
@@ -33,11 +32,41 @@ public class PersistentMap {
void visit(PersistentMapDiskNode node, int depth);
}
public static final Function<byte[], String> STRING_DECODER = t -> new String(t, StandardCharsets.UTF_8);
public static final Function<byte[], String> LONG_DECODER = t -> String
.valueOf(VariableByteEncoder.decodeFirstValue(t));
public interface EncoderDecoder<O> {
public byte[] encode(O object);
public O decode(byte[] bytes);
}
private static final class StringCoder implements EncoderDecoder<String> {
@Override
public byte[] encode(final String object) {
return object.getBytes(StandardCharsets.UTF_8);
}
@Override
public String decode(final byte[] bytes) {
return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8);
}
}
private static final class LongCoder implements EncoderDecoder<Long> {
@Override
public byte[] encode(final Long object) {
return VariableByteEncoder.encode(object);
}
@Override
public Long decode(final byte[] bytes) {
return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes);
}
}
public static final EncoderDecoder<Long> LONG_CODER = new LongCoder();
public static final EncoderDecoder<String> STRING_CODER = new StringCoder();
private static final Charset UTF8 = StandardCharsets.UTF_8;
static final int BLOCK_SIZE = 4096;
static final long NODE_OFFSET_TO_ROOT_NODE = 8;
@@ -45,8 +74,15 @@ public class PersistentMap {
private int maxEntriesInNode = Integer.MAX_VALUE;
public PersistentMap(final DiskStorage diskStore) throws IOException {
private final EncoderDecoder<K> keyEncoder;
private final EncoderDecoder<V> valueEncoder;
public PersistentMap(final DiskStorage diskStore, final EncoderDecoder<K> keyEncoder,
final EncoderDecoder<V> valueEncoder) throws IOException {
this.diskStore = diskStore;
this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder;
initIfNew();
}
@@ -77,51 +113,26 @@ public class PersistentMap {
}
}
public Long putValue(final String key, final long value) throws IOException {
final byte[] oldValue = putValue(key.getBytes(UTF8), VariableByteEncoder.encode(value));
return oldValue == null ? null : VariableByteEncoder.decodeFirstValue(oldValue);
public V putValue(final K key, final V value) throws IOException {
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] encodedValue = valueEncoder.encode(value);
final byte[] oldValue = putValue(encodedKey, encodedValue);
return valueEncoder.decode(oldValue);
}
public String putValue(final long key, final String value) throws IOException {
final byte[] oldValue = putValue(VariableByteEncoder.encode(key), value.getBytes(UTF8));
return oldValue == null ? null : new String(oldValue, UTF8);
public V getValue(final K key) throws IOException {
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] foundValue = getValue(encodedKey);
return valueEncoder.decode(foundValue);
}
public Long putValue(final long key, final long value) throws IOException {
final byte[] oldValue = putValue(VariableByteEncoder.encode(key), VariableByteEncoder.encode(value));
return oldValue == null ? null : VariableByteEncoder.decodeFirstValue(oldValue);
}
public Long getAsLong(final String key) throws IOException {
final byte[] buffer = get(key.getBytes(UTF8));
return buffer == null ? null : VariableByteEncoder.decodeFirstValue(buffer);
}
public Long getAsLong(final long key) throws IOException {
final byte[] buffer = get(VariableByteEncoder.encode(key));
return buffer == null ? null : VariableByteEncoder.decodeFirstValue(buffer);
}
public String putValue(final String key, final String value) throws IOException {
final byte[] keyBytes = key.getBytes(UTF8);
final byte[] valueBytes = value.getBytes(UTF8);
final byte[] oldValue = putValue(keyBytes, valueBytes);
return oldValue == null ? null : new String(oldValue, UTF8);
}
public String getAsString(final String key) throws IOException {
final byte[] value = get(key.getBytes(UTF8));
return value == null ? null : new String(value, UTF8);
}
public byte[] putValue(final byte[] key, final byte[] value) throws IOException {
private byte[] putValue(final byte[] key, final byte[] value) throws IOException {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final Stack<PersistentMapDiskNode> parents = new Stack<>();
return insert(parents, rootNodeOffset, key, value);
}
public byte[] get(final byte[] key) throws IOException {
private byte[] getValue(final byte[] key) throws IOException {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final NodeEntry entry = findNodeEntry(rootNodeOffset, key);
@@ -266,8 +277,7 @@ public class PersistentMap {
diskBlock.force();
}
public void print(final Function<byte[], String> keyDecoder, final Function<byte[], String> valueDecoder)
throws IOException {
public void print() throws IOException {
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
@@ -275,8 +285,8 @@ public class PersistentMap {
final String children = "#" + node.getEntries().size();
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " "
+ nodeEntry.toString(keyDecoder, valueDecoder));
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry
.toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b))));
});
}

30
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java
View File

@@ -15,26 +15,23 @@ import org.lucares.utils.Preconditions;
* <pre>
* Node layout:
*
* ◀────────── Prefix ──────────▶ ◀───────────────── Suffix ──────────────────▶
* ┏━━━━━┳━━━┳━━━━━┳━━━━━┳━━━━━┳━━━┳╸╺╸╺╸╺╸╺┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
* ┃ 456 ┃ 6 ┃ 5,6 ┃ 3,6 ┃ 3,2 ┃ ∅ ┃ ┃"ba"->"147"┃"foobar"->"467"┃"foobaz"->"value"┃
* ┗━━━━━┻━━━┻━━━━━┻━━━━━┻━━━━━┻━━━┻╸╺╸╺╸╺╸╺┻━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━┛
* │ │ │ │ │ │ │ │ └▶ null byte that serves as a separator for the prefix.
* │ │ │ │ │ │ │ └▶ size of the third last key ("ba" in this example)
* │ │ │ │ │ │ └▶ size of the third last value ("147" in this example)
* │ │ │ │ │ └▶ size of the second last key ("foobar" in this example)
* │ │ │ │ └▶ size of the second last value ("467" in this example)
* │ │ │ └▶ size of the last key ("foobaz" in this example)
* │ │ └▶ size of the last value (the string "value" in this example)
* │ └▶ number of entries * 2
* └▶ node offset of the parent node (-1 if there is no parent node)
* ◀─────── Prefix ──────▶ ◀───────────────── Suffix ──────────────────▶
* ┏━━━┳━━━━━┳━━━━━┳━━━━━┳━━━┳╸╺╸╺╸╺╸╺┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
* ┃ 6 ┃ 5,6 ┃ 3,6 ┃ 3,2 ┃ ∅ ┃ ┃"ba"->"147"┃"foobar"->"467"┃"foobaz"->"value"┃
* ┗━━━┻━━━━━┻━━━━━┻━━━━━┻━━━┻╸╺╸╺╸╺╸╺┻━━━━━━━━━━━┻━━━━━━━━━━━━━━━┻━━━━━━━━━━━━━━━━━┛
* │ │ │ │ │ │ │ └▶ null byte that serves as a separator for the prefix.
* │ │ │ │ │ │ └▶ size of the third last key ("ba" in this example)
* │ │ │ │ │ └▶ size of the third last value ("147" in this example)
* │ │ │ │ └▶ size of the second last key ("foobar" in this example)
* │ │ │ └▶ size of the second last value ("467" in this example)
* │ │ └▶ size of the last key ("foobaz" in this example)
* │ └▶ size of the last value (the string "value" in this example)
* └▶ number of entries * 2
*
* </pre>
*/
public class PersistentMapDiskNode {
public static final long NO_NODE_OFFSET = -1;
private final List<NodeEntry> entries;
private final long nodeOffset;
@@ -53,9 +50,6 @@ public class PersistentMapDiskNode {
"block size must be " + PersistentMap.BLOCK_SIZE + " but was " + data.length);
}
final LongList longs = VariableByteEncoder.decode(data);
if (longs.size() == 0) {
System.out.println();
}
final List<NodeEntry> entries = deserialize(longs, data);
return new PersistentMapDiskNode(nodeOffset, entries);

52
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -41,18 +41,20 @@ public class PersistentMapTest {
final String key = "key1";
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
Assert.assertEquals(map.getAsString(key), value);
Assert.assertEquals(map.getValue(key), value);
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
Assert.assertEquals(map.getAsString(key), value);
Assert.assertEquals(map.getValue(key), value);
}
}
@@ -63,7 +65,8 @@ public class PersistentMapTest {
final Random rnd = new Random(1);
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
map.setMaxEntriesInNode(2);
for (int i = 0; i < 100; i++) {
@@ -72,7 +75,7 @@ public class PersistentMapTest {
final String key = nextUUID.toString() + "__" + i;
final String value = "long value to waste some bytes " + i + "__"
+ UUID.randomUUID().toString().repeat(1);
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
@@ -83,10 +86,10 @@ public class PersistentMapTest {
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
final String actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
@@ -97,7 +100,8 @@ public class PersistentMapTest {
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder(
@@ -109,7 +113,7 @@ public class PersistentMapTest {
+ maxDepth.get());
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
@@ -126,14 +130,15 @@ public class PersistentMapTest {
rnd.setSeed(1);
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<Long, Long> map = new PersistentMap<>(ds, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER);
for (int i = 0; i < 1000; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getAsLong(key));
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
@@ -144,10 +149,10 @@ public class PersistentMapTest {
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getAsLong(entry.getKey());
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
@@ -158,7 +163,8 @@ public class PersistentMapTest {
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<Long, Long> map = new PersistentMap<>(ds, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER);
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
@@ -170,7 +176,7 @@ public class PersistentMapTest {
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getAsLong(entry.getKey());
final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
@@ -186,7 +192,8 @@ public class PersistentMapTest {
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) {
@@ -196,7 +203,7 @@ public class PersistentMapTest {
final String key = "" + keyNumber;
final String value = "value";
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
@@ -205,7 +212,7 @@ public class PersistentMapTest {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
@@ -214,7 +221,8 @@ public class PersistentMapTest {
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final PersistentMap<String, String> map = new PersistentMap<>(ds, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger counter = new AtomicInteger();
@@ -222,7 +230,7 @@ public class PersistentMapTest {
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}