andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

adding the blocksize to the metadata section

Browse Source
This commit is contained in:
Andreas Huber
2020-10-10 17:03:55 +02:00
parent f9ed27f03b
commit 277dba4c04
2 changed files with 69 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -2,6 +2,7 @@ package org.lucares.pdb.map;
import java.io.IOException; import java.io.IOException;
import java.io.PrintStream; import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@@ -31,6 +32,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class); private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class);
private static final long CURRENT_VERSION = 1;
// the maximum key // the maximum key
static final byte[] MAX_KEY; static final byte[] MAX_KEY;
static { static {
@@ -151,7 +154,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder(); public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder();
static final int BLOCK_SIZE = 4096; static final int BLOCK_SIZE = 4096;
static final long NODE_OFFSET_TO_ROOT_NODE = 8; static final long OFFSET_META_DATA = 8;
private DiskStorage diskStore; private DiskStorage diskStore;
@@ -170,6 +173,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
private final Path path; private final Path path;
private long version;
public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder, public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder,
final EncoderDecoder<V> valueEncoder) { final EncoderDecoder<V> valueEncoder) {
this.path = path; this.path = path;
@@ -178,12 +183,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
this.valueEncoder = valueEncoder; this.valueEncoder = valueEncoder;
initIfNew(); initIfNew();
readOffsetOfRootNode(); readMetaData();
} updateIfNecessary();
private void readOffsetOfRootNode() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
nodeOffsetOfRootNode = diskBlock.getByteBuffer().getLong(0);
} }
@Override @Override
@@ -198,7 +199,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
private void initIfNew() { private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) { if (diskStore.size() < BLOCK_SIZE) {
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize()); final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE, Preconditions.checkEqual(nodeOffsetToRootNode, OFFSET_META_DATA,
"The offset of the pointer to the root node must be at a well known location. " "The offset of the pointer to the root node must be at a well known location. "
+ "Otherwise we would not be able to find it in an already existing file."); + "Otherwise we would not be able to find it in an already existing file.");
@@ -210,14 +211,40 @@ public class PersistentMap<K, V> implements AutoCloseable {
final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset); final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset);
writeNode(rootNode); writeNode(rootNode);
// 4. update pointer to root node // 4. meta data section with pointer to root node and version
writeNodeOffsetOfRootNode(blockOffset); initMetaDataSection(blockOffset);
// 5. insert a dummy entry with a 'maximum' key // 5. insert a dummy entry with a 'maximum' key
putValue(MAX_KEY, valueEncoder.getEmptyValue()); putValue(MAX_KEY, valueEncoder.getEmptyValue());
} }
} }
private void initMetaDataSection(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(OFFSET_META_DATA, diskStore.minAllocationSize());
final ByteBuffer byteBuffer = diskBlock.getByteBuffer();
byteBuffer.putLong(newNodeOffsetToRootNode);
byteBuffer.putLong(CURRENT_VERSION);
byteBuffer.putLong(BLOCK_SIZE);
diskBlock.force();
nodeOffsetOfRootNode = newNodeOffsetToRootNode;
version = CURRENT_VERSION;
}
private void readMetaData() {
final DiskBlock diskBlock = diskStore.getDiskBlock(OFFSET_META_DATA, diskStore.minAllocationSize());
final ByteBuffer byteBuffer = diskBlock.getByteBuffer();
nodeOffsetOfRootNode = byteBuffer.getLong();
version = byteBuffer.getLong();
}
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(OFFSET_META_DATA, diskStore.minAllocationSize());
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode);
diskBlock.force();
nodeOffsetOfRootNode = newNodeOffsetToRootNode;
}
public synchronized void putAllValues(final Map<K, V> map) { public synchronized void putAllValues(final Map<K, V> map) {
for (final Entry<K, V> e : map.entrySet()) { for (final Entry<K, V> e : map.entrySet()) {
putValue(e.getKey(), e.getValue()); putValue(e.getKey(), e.getValue());
@@ -434,7 +461,9 @@ public class PersistentMap<K, V> implements AutoCloseable {
// diskBlock.force(); // makes writing nodes slower by factor 800 (sic!) // diskBlock.force(); // makes writing nodes slower by factor 800 (sic!)
} }
public synchronized void print() { public synchronized void print(final boolean printValues) {
System.out.println("printing nodes:");
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> { visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
@@ -442,8 +471,11 @@ public class PersistentMap<K, V> implements AutoCloseable {
final String children = "#" + node.getEntries().size(); final String children = "#" + node.getEntries().size();
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry if (printValues || nodeEntry.isInnerNode()) {
.toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b)))); writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " "
+ nodeEntry.toString(b -> String.valueOf(keyEncoder.decode(b)),
b -> String.valueOf(valueEncoder.decode(b))));
}
}); });
} }
@@ -507,7 +539,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
public synchronized void reindex() throws IOException { public synchronized void reindex() throws IOException {
final long start = System.nanoTime(); final long start = System.nanoTime();
final AtomicLong countValues = new AtomicLong(); final AtomicLong countValues = new AtomicLong();
LOGGER.info("start reindexing file: {}", path); LOGGER.info("start reindexing file: {}, version: {}, stats before:\n{}", path, version, stats());
final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp"); final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp");
try (PersistentMap<K, V> newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) { try (PersistentMap<K, V> newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) {
@@ -527,11 +559,12 @@ public class PersistentMap<K, V> implements AutoCloseable {
swapFiles(newFile); swapFiles(newFile);
diskStore = new DiskStorage(path, null); diskStore = new DiskStorage(path, null);
readOffsetOfRootNode(); readMetaData();
version = CURRENT_VERSION;
final double durationInMs = (System.nanoTime() - start) / 1_000_000.0; final double durationInMs = (System.nanoTime() - start) / 1_000_000.0;
final double valuesPerSecond = countValues.get() / (durationInMs / 1000); final double valuesPerSecond = countValues.get() / (durationInMs / 1000);
LOGGER.info("done reindexing, took {} ms, {} values, {} values/s", (int) Math.ceil(durationInMs), LOGGER.info("done reindexing, took {} ms, {} values, {} values/s, stats after:\n{}",
countValues.get(), valuesPerSecond); (int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats());
} }
public synchronized PersistentMapStats stats() { public synchronized PersistentMapStats stats() {
@@ -596,11 +629,15 @@ public class PersistentMap<K, V> implements AutoCloseable {
return nodeOffsetOfRootNode; return nodeOffsetOfRootNode;
} }
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) { private void updateIfNecessary() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize()); try {
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode); if (version < 1) {
diskBlock.force(); reindex();
nodeOffsetOfRootNode = newNodeOffsetToRootNode; }
} catch (final IOException e) {
throw new IllegalStateException(
"failed to update " + path + " from version " + version + " to current version", e);
}
} }
} }

20
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -88,7 +88,7 @@ public class PersistentMapTest {
final String actualValue = map.getValue(entry.getKey()); final String actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print(true);
} }
Assertions.assertEquals(entry.getValue(), actualValue, Assertions.assertEquals(entry.getValue(), actualValue,
@@ -149,7 +149,7 @@ public class PersistentMapTest {
final Long actualValue = map.getValue(entry.getKey()); final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print(true);
} }
Assertions.assertEquals(entry.getValue(), actualValue, Assertions.assertEquals(entry.getValue(), actualValue,
@@ -166,8 +166,8 @@ public class PersistentMapTest {
map.visitNodeEntriesPreOrder( map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0)); (node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assertions.assertEquals(4, counter.get(), Assertions.assertEquals(3, counter.get(),
"number of nodes should be small. Any number larger than 4 indicates, " "number of nodes should be small. Any number larger than 3 indicates, "
+ "that new inner nodes are created even though the existing inner " + "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values"); + "nodes could hold the values");
@@ -202,7 +202,7 @@ public class PersistentMapTest {
insertedValues.put(key, value); insertedValues.put(key, value);
// map.print(); // map.print(false);
final boolean failEarly = false; final boolean failEarly = false;
if (failEarly) { if (failEarly) {
@@ -210,7 +210,7 @@ public class PersistentMapTest {
final Empty actualValue = map.getValue(entry.getKey()); final Empty actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print(false);
} }
Assertions.assertEquals(entry.getValue(), actualValue, Assertions.assertEquals(entry.getValue(), actualValue,
@@ -222,13 +222,13 @@ public class PersistentMapTest {
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER, try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) { PersistentMap.EMPTY_ENCODER)) {
map.print(); // map.print(false);
final AtomicInteger counter = new AtomicInteger(); final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder( map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0)); (node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assertions.assertEquals(4, counter.get(), Assertions.assertEquals(3, counter.get(),
"number of nodes should be small. Any number larger than 4 indicates, " "number of nodes should be small. Any number larger than 3 indicates, "
+ "that new inner nodes are created even though the existing inner " + "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values"); + "nodes could hold the values");
@@ -410,7 +410,7 @@ public class PersistentMapTest {
final Long actualValue = map.getValue(entry.getKey()); final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print(true);
} }
Assertions.assertEquals(entry.getValue(), actualValue, Assertions.assertEquals(entry.getValue(), actualValue,