andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

handle IOExceptions earlier

Browse Source
This commit is contained in:
Andreas Huber
2019-03-17 11:13:46 +01:00
parent 5d0ceb112e
commit 95f2f26966
11 changed files with 174 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFile.java
View File

@@ -1,6 +1,5 @@
package org.lucares.pdb.blockstorage;
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Optional;
@@ -60,14 +59,12 @@ public class BSFile implements AutoCloseable {
private final BSFileCustomizer customizer;
BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer)
throws IOException {
BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this(new BSFileDiskBlock(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE)), diskStorage, customizer);
}
BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer)
throws IOException {
BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this.rootDiskBlock = rootDiskBlock;
this.customizer = customizer;
@@ -98,17 +95,17 @@ public class BSFile implements AutoCloseable {
}
public static BSFile existingFile(final long blockNumber, final DiskStorage diskStorage,
final BSFileCustomizer customizer) throws IOException {
final BSFileCustomizer customizer) {
return new BSFile(blockNumber, diskStorage, customizer);
}
public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) throws IOException {
public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) {
final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
LOGGER.trace("create new bsFile={}", rootBlockOffset);
return new BSFile(rootBlockOffset, diskStorage, customizer);
}
public void append(final long value1, final long value2) throws IOException {
public void append(final long value1, final long value2) {
final long val1 = customizer.preProcessWriteValue1(value1);
final long val2 = customizer.preProcessWriteValue2(value2);
@@ -124,7 +121,7 @@ public class BSFile implements AutoCloseable {
dirty = true;
}
public void append(final long value) throws IOException {
public void append(final long value) {
int bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
if (bytesWritten == 0) {
@@ -136,7 +133,7 @@ public class BSFile implements AutoCloseable {
dirty = true;
}
private void flushFullBufferAndCreateNew() throws IOException {
private void flushFullBufferAndCreateNew() {
final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
@@ -214,23 +211,19 @@ public class BSFile implements AutoCloseable {
@Override
public LongList next() {
try {
if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException();
}
final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset);
nextBlockOffset = diskBlock.getNextBlockNumber();
final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf);
return next;
} catch (final IOException e) {
throw new RuntimeException(e);
if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException();
}
final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset);
nextBlockOffset = diskBlock.getNextBlockNumber();
final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf);
return next;
}
private BSFileDiskBlock getDiskBlock(final long blockOffset) throws IOException {
private BSFileDiskBlock getDiskBlock(final long blockOffset) {
final DiskBlock diskBlock = diskStorage.getDiskBlock(blockOffset, BLOCK_SIZE);
return new BSFileDiskBlock(diskBlock);
}

10
block-storage/src/main/java/org/lucares/pdb/blockstorage/TimeSeriesFile.java
View File

@@ -1,6 +1,5 @@
package org.lucares.pdb.blockstorage;
import java.io.IOException;
import java.util.Optional;
import java.util.stream.Stream;
@@ -11,22 +10,21 @@ public class TimeSeriesFile implements AutoCloseable {
private final BSFile bsFile;
private TimeSeriesFile(final BSFile bsFile) throws IOException {
private TimeSeriesFile(final BSFile bsFile) {
this.bsFile = bsFile;
}
public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage)
throws IOException {
public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public static TimeSeriesFile newFile(final DiskStorage diskStorage) throws IOException {
public static TimeSeriesFile newFile(final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.newFile(diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public void appendTimeValue(final long epochMilli, final long value) throws IOException {
public void appendTimeValue(final long epochMilli, final long value) {
bsFile.append(epochMilli, value);
}

87
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorage.java
View File

@@ -25,14 +25,17 @@ public class DiskStorage implements AutoCloseable {
private final FileChannel fileChannel;
public DiskStorage(final Path databaseFile) throws IOException {
public DiskStorage(final Path databaseFile) {
try {
Files.createDirectories(databaseFile.getParent());
Files.createDirectories(databaseFile.getParent());
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE);
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE);
initIfNew();
initIfNew();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
private void initIfNew() throws IOException {
@@ -42,35 +45,45 @@ public class DiskStorage implements AutoCloseable {
}
}
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) throws IOException {
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) {
try {
LOGGER.trace("read block={}", blockOffset);
LOGGER.trace("read block={}", blockOffset);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
return new DiskBlock(blockOffset, byteBuffer);
return new DiskBlock(blockOffset, byteBuffer);
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
@Override
public void close() throws IOException {
fileChannel.force(true);
fileChannel.close();
public void close() {
try {
fileChannel.force(true);
fileChannel.close();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public synchronized long allocateBlock(final int blockSize) throws IOException {
public synchronized long allocateBlock(final int blockSize) {
if (blockSize < FREE_LIST_NODE_SIZE) {
throw new IllegalArgumentException("The minimal allocation size is 32 byte.");
}
final var optionalFreeBlock = findFreeBlockWithSize(blockSize);
if (optionalFreeBlock.isPresent()) {
final FreeListNode freeBlock = optionalFreeBlock.get();
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
try {
final var optionalFreeBlock = findFreeBlockWithSize(blockSize);
if (optionalFreeBlock.isPresent()) {
final FreeListNode freeBlock = optionalFreeBlock.get();
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
@@ -237,18 +250,26 @@ public class DiskStorage implements AutoCloseable {
fileChannel.write(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
}
public synchronized void ensureAlignmentForNewBlocks(final int alignment) throws IOException {
final long size = fileChannel.size();
final int alignmentMismatch = Math.floorMod(size, alignment);
if (alignmentMismatch != 0) {
// The next allocated block would not be aligned. Therefore we allocate a
// throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
public synchronized void ensureAlignmentForNewBlocks(final int alignment) {
try {
final long size = fileChannel.size();
final int alignmentMismatch = Math.floorMod(size, alignment);
if (alignmentMismatch != 0) {
// The next allocated block would not be aligned. Therefore we allocate a
// throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public long size() throws IOException {
return fileChannel.size();
public long size() {
try {
return fileChannel.size();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public int minAllocationSize() {

19
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorageException.java Normal file
View File

@@ -0,0 +1,19 @@
package org.lucares.pdb.diskstorage;
public class DiskStorageException extends RuntimeException {
private static final long serialVersionUID = 1683775743640383633L;
public DiskStorageException(final String message, final Throwable cause) {
super(message, cause);
}
public DiskStorageException(final String message) {
super(message);
}
public DiskStorageException(final Throwable cause) {
super(cause);
}
}

43
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -1,6 +1,5 @@
package org.lucares.pdb.map;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
@@ -126,8 +125,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
private final LRUCache<K, V> valueCache = new LRUCache<>(1_000);
public PersistentMap(final Path path, final EncoderDecoder<K> keyEncoder, final EncoderDecoder<V> valueEncoder)
throws IOException {
public PersistentMap(final Path path, final EncoderDecoder<K> keyEncoder, final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path);
this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder;
@@ -135,7 +133,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
@Override
public void close() throws IOException {
public void close() {
diskStore.close();
}
@@ -143,7 +141,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
this.maxEntriesInNode = maxEntriesInNode;
}
private void initIfNew() throws IOException {
private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) {
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE,
@@ -166,13 +164,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
}
public synchronized void putAllValues(final Map<K, V> map) throws IOException {
public synchronized void putAllValues(final Map<K, V> map) {
for (final Entry<K, V> e : map.entrySet()) {
putValue(e.getKey(), e.getValue());
}
}
public synchronized V putValue(final K key, final V value) throws IOException {
public synchronized V putValue(final K key, final V value) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null && cachedValue == value) {
@@ -187,7 +185,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
return oldValue;
}
public synchronized V getValue(final K key) throws IOException {
public synchronized V getValue(final K key) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null) {
@@ -201,13 +199,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
return result;
}
private byte[] putValue(final byte[] key, final byte[] value) throws IOException {
private byte[] putValue(final byte[] key, final byte[] value) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final Stack<PersistentMapDiskNode> parents = new Stack<>();
return insert(parents, rootNodeOffset, key, value);
}
private byte[] getValue(final byte[] key) throws IOException {
private byte[] getValue(final byte[] key) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final NodeEntry entry = findNodeEntry(rootNodeOffset, key);
@@ -215,7 +213,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
private byte[] insert(final Stack<PersistentMapDiskNode> parents, final long nodeOffest, final byte[] key,
final byte[] value) throws IOException {
final byte[] value) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final NodeEntry entry = node.getNodeEntryTo(key);
@@ -266,7 +264,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
private PersistentMapDiskNode splitNode(final Stack<PersistentMapDiskNode> parents,
final PersistentMapDiskNode node) throws IOException {
final PersistentMapDiskNode node) {
// System.out.println("\n\npre split node: " + node + "\n");
@@ -321,7 +319,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
}
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) throws IOException {
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final var entry = node.getNodeEntryTo(key);
@@ -344,7 +342,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
return VariableByteEncoder.decodeFirstValue(entry.getValue());
}
private PersistentMapDiskNode getNode(final long nodeOffset) throws IOException {
private PersistentMapDiskNode getNode(final long nodeOffset) {
PersistentMapDiskNode node = nodeCache.get(nodeOffset);
if (node == null) {
@@ -358,7 +356,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
return node;
}
private void writeNode(final PersistentMapDiskNode node) throws IOException {
private void writeNode(final PersistentMapDiskNode node) {
LOGGER.trace("writing node {}", node);
final long nodeOffest = node.getNodeOffset();
// final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
@@ -373,7 +371,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
// diskBlock.force(); // makes writing nodes slower by factor 800 (sic!)
}
public synchronized void print() throws IOException {
public synchronized void print() {
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
@@ -386,13 +384,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
});
}
public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) throws IOException {
public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
visitNodeEntriesPreOrderRecursively(rootNodeOffset, null, visitor, 0);
}
private void visitNodeEntriesPreOrderRecursively(final long nodeOffset, final PersistentMapDiskNode parentNode,
final VisitorCallback visitor, final int depth) throws IOException {
final VisitorCallback visitor, final int depth) {
final PersistentMapDiskNode node = getNode(nodeOffset);
for (final NodeEntry child : node.getEntries()) {
@@ -409,15 +407,14 @@ public class PersistentMap<K, V> implements AutoCloseable {
FIND, ITERATE
}
public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) throws IOException {
public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) {
final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix);
final long rootNodeOffset = readNodeOffsetOfRootNode();
iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor);
}
private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor)
throws IOException {
private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) {
final PersistentMapDiskNode node = getNode(nodeOffest);
// list of children that might contain a key with the keyPrefix
@@ -447,13 +444,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
}
}
private long readNodeOffsetOfRootNode() throws IOException {
private long readNodeOffsetOfRootNode() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
return diskBlock.getByteBuffer().getLong(0);
}
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) throws IOException {
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode);
diskBlock.force();