andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

improve trace logging

Browse Source 
- Add filename for trace logs for read/write operations.
This commit is contained in:
Andreas Huber
2019-08-18 09:25:49 +02:00
parent 16bf0933e7
commit 3252fcf42d
15 changed files with 102 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFile.java
View File

@@ -159,7 +159,7 @@ public class BSFile implements AutoCloseable {
public void flush() {
LOGGER.trace("flush bsFile={} dirty={}", rootBlockOffset, dirty);
LOGGER.trace("flush bsFile={} dirty={} file={}", rootBlockOffset, dirty, diskStorage.getRelativeDatabaseFileForLogging());
if (dirty) {
buffer.writeAsync();
}

12
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorage.java
View File

@@ -25,7 +25,11 @@ public class DiskStorage implements AutoCloseable {
private final FileChannel fileChannel;
public DiskStorage(final Path databaseFile) {
private Path relativeDatabaseFileForLogging;
public DiskStorage(final Path databaseFile, Path storageBasePath) {
this.relativeDatabaseFileForLogging = storageBasePath != null ? storageBasePath.relativize(databaseFile): databaseFile;
try {
Files.createDirectories(databaseFile.getParent());
@@ -47,7 +51,7 @@ public class DiskStorage implements AutoCloseable {
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) {
try {
LOGGER.trace("read block={}", blockOffset);
LOGGER.trace("read block={} file={}", blockOffset, relativeDatabaseFileForLogging);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
@@ -56,6 +60,10 @@ public class DiskStorage implements AutoCloseable {
throw new DiskStorageException(e);
}
}
public Path getRelativeDatabaseFileForLogging() {
return relativeDatabaseFileForLogging;
}
@Override
public void close() {

24
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -48,6 +48,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
public default Function<O,byte[]> asEncoder() {
return plain -> this.encode(plain);
}
public byte[] getEmptyValue();
}
private static final class StringCoder implements EncoderDecoder<String> {
@@ -61,6 +63,10 @@ public class PersistentMap<K, V> implements AutoCloseable {
public String decode(final byte[] bytes) {
return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8);
}
public byte[] getEmptyValue() {
return new byte[] {0};
}
}
private static final class LongCoder implements EncoderDecoder<Long> {
@@ -74,6 +80,10 @@ public class PersistentMap<K, V> implements AutoCloseable {
public Long decode(final byte[] bytes) {
return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes);
}
public byte[] getEmptyValue() {
return new byte[] {0};
}
}
private static final class UUIDCoder implements EncoderDecoder<UUID> {
@@ -94,6 +104,10 @@ public class PersistentMap<K, V> implements AutoCloseable {
return new UUID(mostSignificantBits, leastSignificantBits);
}
public byte[] getEmptyValue() {
return new byte[] {0};
}
}
private static final class EmptyCoder implements EncoderDecoder<Empty> {
@@ -112,6 +126,10 @@ public class PersistentMap<K, V> implements AutoCloseable {
return Empty.INSTANCE;
}
public byte[] getEmptyValue() {
return new byte[] {};
}
}
public static final EncoderDecoder<Long> LONG_CODER = new LongCoder();
@@ -134,8 +152,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
private final LRUCache<K, V> valueCache = new LRUCache<>(1_000);
public PersistentMap(final Path path, final EncoderDecoder<K> keyEncoder, final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path);
public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder, final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path, storageBasePath);
this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder;
initIfNew();
@@ -169,7 +187,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
writeNodeOffsetOfRootNode(blockOffset);
// 5. insert a dummy entry with a 'maximum' key
putValue(MAX_KEY, new byte[] { });
putValue(MAX_KEY, valueEncoder.getEmptyValue());
}
}

10
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -43,8 +43,8 @@ public class BSFileTest {
long blockOffset = -1;
long start = System.nanoTime();
//
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
@@ -64,7 +64,7 @@ public class BSFileTest {
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
@@ -83,7 +83,7 @@ public class BSFileTest {
final Map<Long, LongList> expected = new HashMap<>();
final List<Future<Void>> futures = new ArrayList<>();
final long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
for (int i = 0; i < threads; i++) {
final Future<Void> future = pool.submit(() -> {
@@ -117,7 +117,7 @@ public class BSFileTest {
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
// verification
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
for (final Entry<Long, LongList> entry : expected.entrySet()) {
final long rootBlockNumber = entry.getKey();
final LongList expectedValues = entry.getValue();

4
block-storage/src/test/java/org/lucares/pdb/blockstorage/TimeSeriesFileTest.java
View File

@@ -37,7 +37,7 @@ public class TimeSeriesFileTest {
long start = System.nanoTime();
long lastEpochMilli = 0;
//
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) {
@@ -72,7 +72,7 @@ public class TimeSeriesFileTest {
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList();

16
block-storage/src/test/java/org/lucares/pdb/diskstorage/DiskStorageTest.java
View File

@@ -43,7 +43,7 @@ public class DiskStorageTest {
final Path databaseFile = dataDirectory.resolve("db.ds");
Files.deleteIfExists(databaseFile);
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
allocateBlocks(ds, numBlocks, BLOCK_SIZE);
@@ -81,7 +81,7 @@ public class DiskStorageTest {
//
// But it does see the changes. Most likely, because both channels
// use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile)) {
try (DiskStorage ds2 = new DiskStorage(databaseFile, dataDirectory)) {
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
@@ -98,7 +98,7 @@ public class DiskStorageTest {
final ExecutorService pool = Executors.newCachedThreadPool();
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
final long[] blockOffsets = allocateBlocks(ds, numBlocks, BLOCK_SIZE);
@@ -140,7 +140,7 @@ public class DiskStorageTest {
public void testAllocationSmallerThanMinimalBlockSize() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 31; // minimal block size is 32
ds.allocateBlock(blockSize);
@@ -151,7 +151,7 @@ public class DiskStorageTest {
public void testAllocateAndFreeSingleBlockInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
final long block_8_39 = ds.allocateBlock(blockSize);
@@ -175,7 +175,7 @@ public class DiskStorageTest {
public void testAllocateAndFreeMultipleBlocksInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
ds.allocateBlock(blockSize);
@@ -212,7 +212,7 @@ public class DiskStorageTest {
public void testAllocateAndFreeInsertFreeNodeInTheMiddleOfTheFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
ds.allocateBlock(blockSize);
@@ -242,7 +242,7 @@ public class DiskStorageTest {
public void testAllocateAndFreeMultipleBlocksWithDifferentSizes() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSizeSmall = 32;
final int blockSizeBig = 64;

28
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -41,7 +41,7 @@ public class PersistentMapTest {
final String value = "value1";
final String key = "key1";
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
Assert.assertNull(map.getValue(key));
@@ -50,7 +50,7 @@ public class PersistentMapTest {
Assert.assertEquals(map.getValue(key), value);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
Assert.assertEquals(map.getValue(key), value);
@@ -64,7 +64,7 @@ public class PersistentMapTest {
final Random rnd = new Random(1);
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
map.setMaxEntriesInNode(2);
@@ -98,7 +98,7 @@ public class PersistentMapTest {
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger();
@@ -127,7 +127,7 @@ public class PersistentMapTest {
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1000; i++) {
@@ -159,7 +159,7 @@ public class PersistentMapTest {
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger();
@@ -187,7 +187,7 @@ public class PersistentMapTest {
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
for (int i = 0; i < 1500; i++) {
@@ -219,7 +219,7 @@ public class PersistentMapTest {
}
}
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
map.print();
final AtomicInteger counter = new AtomicInteger();
@@ -247,7 +247,7 @@ public class PersistentMapTest {
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
final int numbersSize = numbers.size();
@@ -275,7 +275,7 @@ public class PersistentMapTest {
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
@@ -309,13 +309,13 @@ public class PersistentMapTest {
input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
map.putAllValues(input);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, PersistentMap.STRING_CODER,
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
{
@@ -336,7 +336,7 @@ public class PersistentMapTest {
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1_000; i++) {
@@ -368,7 +368,7 @@ public class PersistentMapTest {
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, PersistentMap.LONG_CODER,
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
final AtomicInteger counter = new AtomicInteger();
final AtomicInteger maxDepth = new AtomicInteger();