andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use byte offsets instead of block numbers

Browse Source 
We want to allow arbitrary allocations in DiskStorage. The
first step was to change the hard coded block size into a
dynamic one.
This commit is contained in:
Andreas Huber
2018-10-12 08:10:43 +02:00
parent eaa234bfa5
commit 0539080200
6 changed files with 96 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -41,7 +41,7 @@ public class BSFileTest {
public void testBlockStorage() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final int numLongs = 1000;
long blockNumber = -1;
long blockOffset = -1;
long start = System.nanoTime();
//
@@ -49,13 +49,13 @@ public class BSFileTest {
try (final BSFile bsFile = BSFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockNumber();
blockOffset = bsFile.getRootBlockOffset();
for (long i = 0; i < numLongs / 2; i++) {
bsFile.append(i);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockNumber, ds)) {
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds)) {
for (long i = numLongs / 2; i < numLongs; i++) {
bsFile.append(i);
@@ -66,7 +66,7 @@ public class BSFileTest {
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
final BSFile bsFile = BSFile.existingFile(blockNumber, ds);
final BSFile bsFile = BSFile.existingFile(blockOffset, ds);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs);
@@ -100,7 +100,7 @@ public class BSFileTest {
listOfValues.add(value);
bsFile.append(value);
}
expected.put(bsFile.getRootBlockNumber(), listOfValues);
expected.put(bsFile.getRootBlockOffset(), listOfValues);
}
return null;
@@ -146,7 +146,7 @@ public class BSFileTest {
try (final BSFile bsFile = BSFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockNumber();
blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) {

26
block-storage/src/test/java/org/lucares/pdb/diskstorage/DiskStorageTest.java
View File

@@ -11,13 +11,13 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class DiskStorageTest {
private static final int BLOCK_SIZE = 512;
private Path dataDirectory;
@@ -45,14 +45,13 @@ public class DiskStorageTest {
try (DiskStorage ds = new DiskStorage(databaseFile)) {
final int numBlocks = 10;
ds.appendNewBlocks(numBlocks);
Assert.assertEquals(ds.getNumBlocks(), numBlocks);
ds.allocateBlocks(numBlocks, BLOCK_SIZE);
final List<DiskBlock> blocks = new ArrayList<>();
// fill the first 16 512-byte blocks
// that is more than on 4096 byte block
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i);
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) i);
diskBlock.writeAsync();
@@ -70,7 +69,7 @@ public class DiskStorageTest {
// 1. we do this with the existing file channel
// this one should see every change, because we wrote them to the file channel
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i);
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
@@ -83,7 +82,7 @@ public class DiskStorageTest {
// use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile)) {
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i);
final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
@@ -92,7 +91,7 @@ public class DiskStorageTest {
}
}
@Test(enabled = false)
@Test(enabled = true)
public void testDiskStorage() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@@ -101,18 +100,17 @@ public class DiskStorageTest {
try (DiskStorage ds = new DiskStorage(databaseFile)) {
final int numBlocks = 10;
ds.appendNewBlocks(numBlocks);
Assert.assertEquals(ds.getNumBlocks(), numBlocks);
final long[] blockOffsets = ds.allocateBlocks(numBlocks, BLOCK_SIZE);
for (int i = 0; i < numBlocks; i++) {
for (final long blockOffset : blockOffsets) {
final int block = i;
final long block = blockOffset;
pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
try {
// now read/write random blocks
for (int j = 0; j < 10; j++) {
final DiskBlock diskBlock = ds.getDiskBlock(block);
final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) random.nextInt(127));
@@ -142,7 +140,7 @@ public class DiskStorageTest {
for (int i = 0; i < buffer.length; i++) {
if (expectedVal != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockNumber() + " " + buffer[i] + " != " + expectedVal + " at " + i);
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i);
break;
}
}
@@ -155,7 +153,7 @@ public class DiskStorageTest {
for (int i = 0; i < buffer.length; i++) {
if (expected != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockNumber() + " " + buffer[i] + " != " + expected + " at " + i);
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i);
break;
}
}