andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

insertion of many values into the persistent map

Browse Source
This commit is contained in:
Andreas Huber
2018-11-04 10:11:10 +01:00
parent c6782df0e5
commit f2d5c27668
8 changed files with 467 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java
View File

@@ -1,32 +1,7 @@
package org.lucares.pdb.map;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.lucares.pdb.map.NodeEntry.ValueType;
import org.testng.Assert;
import org.testng.annotations.Test;
@Test
public class NodeEntryTest {
public void serializeDeserialize() throws Exception {
final List<NodeEntry> entries = new ArrayList<>();
entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1"));
entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--"));
entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---"));
entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----"));
final byte[] buffer = NodeEntry.serialize(entries);
final List<NodeEntry> actualEntries = NodeEntry.deserialize(buffer);
Assert.assertEquals(actualEntries, entries);
}
private static NodeEntry newNode(final ValueType type, final String key, final String value) {
return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8),
value.getBytes(StandardCharsets.UTF_8));
}
}

38
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapDiskNodeTest.java Normal file
View File

@@ -0,0 +1,38 @@
package org.lucares.pdb.map;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import org.lucares.pdb.map.NodeEntry.ValueType;
import org.testng.Assert;
import org.testng.annotations.Test;
@Test
public class PersistentMapDiskNodeTest {
public void serializeDeserialize() throws Exception {
final List<NodeEntry> entries = new ArrayList<>();
entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1"));
entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--"));
entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---"));
entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----"));
final long nodeOffset = ThreadLocalRandom.current().nextInt();
final PersistentMapDiskNode node = new PersistentMapDiskNode(nodeOffset, entries);
final byte[] buffer = node.serialize();
final PersistentMapDiskNode actualNode = PersistentMapDiskNode.parse(nodeOffset, buffer);
Assert.assertEquals(actualNode.getEntries(), entries);
}
private static NodeEntry newNode(final ValueType type, final String key, final String value) {
return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8),
value.getBytes(StandardCharsets.UTF_8));
}
}

174
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -1,10 +1,15 @@
package org.lucares.pdb.map;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Objects;
import java.util.Queue;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
@@ -40,7 +45,7 @@ public class PersistentMapTest {
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.put(key, value));
Assert.assertNull(map.putValue(key, value));
Assert.assertEquals(map.getAsString(key), value);
}
@@ -55,21 +60,153 @@ public class PersistentMapTest {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Random rnd = new Random(1);
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
map.setMaxEntriesInNode(2);
for (int i = 0; i < 200; i++) {
final String key = UUID.randomUUID().toString() + "__" + i;
final String value = "long value to waste some bytes " + i;
for (int i = 0; i < 100; i++) {
// System.out.println("\n\ninserting: " + i);
final UUID nextUUID = new UUID(rnd.nextLong(), rnd.nextLong());
final String key = nextUUID.toString() + "__" + i;
final String value = "long value to waste some bytes " + i + "__"
+ UUID.randomUUID().toString().repeat(1);
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.put(key, value));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> maxDepth.set(Math.max(depth, maxDepth.get())));
Assert.assertTrue(maxDepth.get() >= 4,
"The tree's depth. This test must have at least depth 4, "
+ "so that we can be sure that splitting parent nodes works recursively, but was "
+ maxDepth.get());
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test
public void testManySmallValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
for (int i = 0; i < 1000; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getAsLong(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print();
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getAsLong(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getAsLong(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testEasyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) {
final Integer keyNumber = numbers.poll();
// System.out.println("\n\ninserting: " + keyNumber);
final String key = "" + keyNumber;
final String value = "value";
Assert.assertNull(map.getAsString(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
@@ -78,28 +215,11 @@ public class PersistentMapTest {
try (final DiskStorage ds = new DiskStorage(file)) {
final PersistentMap map = new PersistentMap(ds);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
map.visitNodeEntriesPreOrder((nodeEntry, depth) -> {
if (nodeEntry.isInnerNode()) {
System.out.println(" ".repeat(depth) + nodeEntry);
} else {
System.out.println(" ".repeat(depth) + nodeEntry);
}
});
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder((nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
System.out.println(" -------------");
map.visitNodesPreOrder((node, depth) -> {
final String key = new String(node.getTopNodeEntry().getKey(), StandardCharsets.UTF_8);
System.out.println(" ".repeat(depth) + node.getNodeOffset() + " " + key + " (children: "
+ node.getEntries().size() + ")");
});
// Assert.assertEquals(counter.get(), 3,
// "number of nodes should be small. Any number larger than 3 indicates, "
// + "that new inner nodes are created even though the existing inner "
// + "nodes could hold the values");
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getAsString(entry.getKey());