andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use byte offsets instead of block numbers

Browse Source 
We want to allow arbitrary allocations in DiskStorage. The
first step was to change the hard coded block size into a
dynamic one.
This commit is contained in:
Andreas Huber
2018-10-12 08:10:43 +02:00
parent eaa234bfa5
commit 0539080200
6 changed files with 96 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFile.java
View File

@@ -59,6 +59,8 @@ public class BSFile implements AutoCloseable {
private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder(); private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder();
public static final int BLOCK_SIZE = 512;
/* /*
* The last disk block of this sequence. This is the block new values will be * The last disk block of this sequence. This is the block new values will be
* appended to. * appended to.
@@ -69,7 +71,7 @@ public class BSFile implements AutoCloseable {
private boolean dirty = false; private boolean dirty = false;
private final long rootBlockNumber; private final long rootBlockOffset;
private final DiskStorage diskStorage; private final DiskStorage diskStorage;
@@ -77,26 +79,26 @@ public class BSFile implements AutoCloseable {
private long lastEpochMilli; private long lastEpochMilli;
BSFile(final long rootBlockNumber, final DiskStorage diskStorage) throws IOException { BSFile(final long rootBlockOffset, final DiskStorage diskStorage) throws IOException {
this(diskStorage.getDiskBlock(rootBlockNumber), diskStorage); this(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE), diskStorage);
} }
BSFile(final DiskBlock rootDiskBlock, final DiskStorage diskStorage) throws IOException { BSFile(final DiskBlock rootDiskBlock, final DiskStorage diskStorage) throws IOException {
this.rootDiskBlock = rootDiskBlock; this.rootDiskBlock = rootDiskBlock;
this.rootBlockNumber = rootDiskBlock.getBlockNumber(); this.rootBlockOffset = rootDiskBlock.getBlockOffset();
this.diskStorage = diskStorage; this.diskStorage = diskStorage;
final long lastBlockNumber = rootDiskBlock.getLastBlockPointer(); final long lastBlockNumber = rootDiskBlock.getLastBlockPointer();
if (lastBlockNumber == rootBlockNumber || lastBlockNumber == 0) { if (lastBlockNumber == rootBlockOffset || lastBlockNumber == 0) {
buffer = rootDiskBlock; buffer = rootDiskBlock;
} else { } else {
buffer = diskStorage.getDiskBlock(lastBlockNumber); buffer = diskStorage.getDiskBlock(lastBlockNumber, BLOCK_SIZE);
} }
offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer); offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer);
lastEpochMilli = determineLastEpochMilli(buffer); lastEpochMilli = determineLastEpochMilli(buffer);
LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockNumber, lastBlockNumber); LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockOffset, lastBlockNumber);
} }
private long determineLastEpochMilli(final DiskBlock diskBlock) { private long determineLastEpochMilli(final DiskBlock diskBlock) {
@@ -136,9 +138,9 @@ public class BSFile implements AutoCloseable {
} }
public static BSFile newFile(final DiskStorage diskStorage) throws IOException { public static BSFile newFile(final DiskStorage diskStorage) throws IOException {
final long rootBlockNumber = diskStorage.appendNewBlock(); final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
LOGGER.trace("create new bsFile={}", rootBlockNumber); LOGGER.trace("create new bsFile={}", rootBlockOffset);
return new BSFile(rootBlockNumber, diskStorage); return new BSFile(rootBlockOffset, diskStorage);
} }
public void appendTimeValue(final long epochMilli, final long value) throws IOException { public void appendTimeValue(final long epochMilli, final long value) throws IOException {
@@ -176,33 +178,31 @@ public class BSFile implements AutoCloseable {
private void flushFullBufferAndCreateNew() throws IOException { private void flushFullBufferAndCreateNew() throws IOException {
final long start = System.nanoTime(); final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
final long newBlockNumber = diskStorage.appendNewBlock();
if (buffer == rootDiskBlock) { if (buffer == rootDiskBlock) {
// root block and current block are the same, so we need // root block and current block are the same, so we need
// to update only one // to update only one
buffer.setLastBlockNumber(newBlockNumber); buffer.setLastBlockOffset(newBlockOffset);
buffer.setNextBlockNumber(newBlockNumber); buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync(); buffer.writeAsync();
} else { } else {
rootDiskBlock.writeLastBlockNumber(newBlockNumber); rootDiskBlock.writeLastBlockOffset(newBlockOffset);
buffer.setNextBlockNumber(newBlockNumber); buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync(); buffer.writeAsync();
} }
// set the new buffer // set the new buffer
buffer = diskStorage.getDiskBlock(newBlockNumber); buffer = diskStorage.getDiskBlock(newBlockOffset, BLOCK_SIZE);
offsetInBuffer = 0; offsetInBuffer = 0;
dirty = false; dirty = false;
LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}: {}ms", rootBlockNumber, newBlockNumber, LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}", rootBlockOffset, newBlockOffset);
(System.nanoTime() - start) / 1_000_000.0);
} }
public void flush() { public void flush() {
LOGGER.trace("flush bsFile={} dirty={}", rootBlockNumber, dirty); LOGGER.trace("flush bsFile={} dirty={}", rootBlockOffset, dirty);
if (dirty) { if (dirty) {
buffer.writeAsync(); buffer.writeAsync();
} }
@@ -224,7 +224,7 @@ public class BSFile implements AutoCloseable {
} }
public Stream<LongList> streamOfLongLists() { public Stream<LongList> streamOfLongLists() {
final Iterator<LongList> iterator = new LongListIterator(rootBlockNumber, diskStorage); final Iterator<LongList> iterator = new LongListIterator(rootBlockOffset, diskStorage);
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
} }
@@ -236,29 +236,29 @@ public class BSFile implements AutoCloseable {
private static class LongListIterator implements Iterator<LongList> { private static class LongListIterator implements Iterator<LongList> {
private LongList next = null; private LongList next = null;
private long nextBlockNumber; private long nextBlockOffset;
private final DiskStorage diskStorage; private final DiskStorage diskStorage;
public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) { public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) {
this.nextBlockNumber = nextBlockNumber; this.nextBlockOffset = nextBlockNumber;
this.diskStorage = diskStorage; this.diskStorage = diskStorage;
} }
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return nextBlockNumber != DiskBlock.NO_NEXT_POINTER; return nextBlockOffset != DiskBlock.NO_NEXT_POINTER;
} }
@Override @Override
public LongList next() { public LongList next() {
try { try {
if (nextBlockNumber == DiskBlock.NO_NEXT_POINTER) { if (nextBlockOffset == DiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException(); throw new NoSuchElementException();
} }
final DiskBlock diskBlock = diskStorage.getDiskBlock(nextBlockNumber); final DiskBlock diskBlock = diskStorage.getDiskBlock(nextBlockOffset, BLOCK_SIZE);
nextBlockNumber = diskBlock.getNextBlockNumber(); nextBlockOffset = diskBlock.getNextBlockNumber();
final byte[] buf = diskBlock.getBuffer(); final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf); next = VariableByteEncoder.decode(buf);
@@ -283,9 +283,9 @@ public class BSFile implements AutoCloseable {
return result; return result;
} }
public long getRootBlockNumber() { public long getRootBlockOffset() {
return rootBlockNumber; return rootBlockOffset;
} }
@Override @Override

50
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskBlock.java
View File

@@ -19,14 +19,14 @@ public class DiskBlock {
+ 8; // last block pointer; + 8; // last block pointer;
private byte[] buffer = null; private byte[] buffer = null;
private final long blockNumber; private final long blockOffset;
private long nextBlockNumber = 0; private long nextBlockOffset = 0;
private long lastBlockNumber = 0; private long lastBlockOffset = 0;
private final MappedByteBuffer byteBuffer; private final MappedByteBuffer byteBuffer;
public DiskBlock(final long blockNumber, final MappedByteBuffer byteBuffer) { public DiskBlock(final long blockOffset, final MappedByteBuffer byteBuffer) {
this.blockNumber = blockNumber; this.blockOffset = blockOffset;
this.byteBuffer = byteBuffer; this.byteBuffer = byteBuffer;
} }
@@ -41,12 +41,12 @@ public class DiskBlock {
return buffer; return buffer;
} }
public long getBlockNumber() { public long getBlockOffset() {
return blockNumber; return blockOffset;
} }
public void setNextBlockNumber(final long nextBlockNumber) { public void setNextBlockOffset(final long nextBlockOffset) {
this.nextBlockNumber = nextBlockNumber; this.nextBlockOffset = nextBlockOffset;
} }
private void writeBufferToByteBuffer() { private void writeBufferToByteBuffer() {
@@ -55,8 +55,8 @@ public class DiskBlock {
} }
private void writeBlockHeader() { private void writeBlockHeader() {
byteBuffer.putLong(NEXT_POINTER_OFFSET, nextBlockNumber); byteBuffer.putLong(NEXT_POINTER_OFFSET, nextBlockOffset);
byteBuffer.putLong(LAST_BLOCK_POINTER_POSITION, lastBlockNumber); byteBuffer.putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
} }
public void writeAsync() { public void writeAsync() {
@@ -64,43 +64,43 @@ public class DiskBlock {
writeBlockHeader(); writeBlockHeader();
writeBufferToByteBuffer(); writeBufferToByteBuffer();
final long duration = System.nanoTime() - start; final long duration = System.nanoTime() - start;
LOGGER.trace("write() of block={}: {}ms", blockNumber, duration / 1_000_000.0); LOGGER.trace("write() of block={}: {}ms", blockOffset, duration / 1_000_000.0);
} }
public void force() { public void force() {
final long start = System.nanoTime(); final long start = System.nanoTime();
byteBuffer.force(); byteBuffer.force();
LOGGER.trace("force of block={}: {}ms", blockNumber, (System.nanoTime() - start) / 1_000_000.0); LOGGER.trace("force of block={}: {}ms", blockOffset, (System.nanoTime() - start) / 1_000_000.0);
} }
public long getLastBlockPointer() { public long getLastBlockPointer() {
if (lastBlockNumber <= 0) { if (lastBlockOffset <= 0) {
lastBlockNumber = byteBuffer.getLong(LAST_BLOCK_POINTER_POSITION); lastBlockOffset = byteBuffer.getLong(LAST_BLOCK_POINTER_POSITION);
} }
return lastBlockNumber; return lastBlockOffset;
} }
public long getNextBlockNumber() { public long getNextBlockNumber() {
if (nextBlockNumber <= 0) { if (nextBlockOffset <= 0) {
nextBlockNumber = byteBuffer.getLong(NEXT_POINTER_OFFSET); nextBlockOffset = byteBuffer.getLong(NEXT_POINTER_OFFSET);
} }
return nextBlockNumber; return nextBlockOffset;
} }
public void setLastBlockNumber(final long lastBlockNumber) { public void setLastBlockOffset(final long lastBlockOffset) {
this.lastBlockNumber = lastBlockNumber; this.lastBlockOffset = lastBlockOffset;
} }
public void writeLastBlockNumber(final long lastBlockNumber) { public void writeLastBlockOffset(final long lastBlockOffset) {
this.lastBlockNumber = lastBlockNumber; this.lastBlockOffset = lastBlockOffset;
byteBuffer.putLong(LAST_BLOCK_POINTER_POSITION, lastBlockNumber); byteBuffer.putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
} }
@Override @Override
public String toString() { public String toString() {
final LongList bufferDecoded = VariableByteEncoder.decode(buffer); final LongList bufferDecoded = VariableByteEncoder.decode(buffer);
return "DiskBlock[" + blockNumber + ", bufferDecoded=" + bufferDecoded + "]"; return "DiskBlock[" + blockOffset + ", bufferDecoded=" + bufferDecoded + "]";
} }
} }

48
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorage.java
View File

@@ -17,8 +17,6 @@ public class DiskStorage implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class); private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class);
public static final int BLOCK_SIZE = 512;
private final FileChannel fileChannel; private final FileChannel fileChannel;
public DiskStorage(final Path databaseFile) throws IOException { public DiskStorage(final Path databaseFile) throws IOException {
@@ -27,25 +25,27 @@ public class DiskStorage implements AutoCloseable {
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE, fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE); StandardOpenOption.CREATE);
if (fileChannel.size() == 0) {
// file is new -> add root of the free list
// TODO implement a real free list
final ByteBuffer src = ByteBuffer.allocate(8);
fileChannel.write(src, 0);
}
} }
public DiskBlock getDiskBlock(final long blockNumber) throws IOException { public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) throws IOException {
// block numbers start with 1, so that the uninitialized value
// (0) means 'no block'. That way we do not have to write data to a newly
// created block, which reduces IO.
final long position = (blockNumber - 1) * BLOCK_SIZE;
final long start = System.nanoTime(); final long start = System.nanoTime();
try (final FileLock lock = fileChannel.lock(position, BLOCK_SIZE, true)) { try (final FileLock lock = fileChannel.lock(blockOffset, blockSize, true)) {
final MappedByteBuffer byteBuffer = fileChannel.map(MapMode.READ_WRITE, position, BLOCK_SIZE); final MappedByteBuffer byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
return new DiskBlock(blockNumber, byteBuffer); return new DiskBlock(blockOffset, byteBuffer);
} finally { } finally {
LOGGER.trace("read block={}: {}ms", blockNumber, (System.nanoTime() - start) / 1_000_000.0); LOGGER.trace("read block={}: {}ms", blockOffset, (System.nanoTime() - start) / 1_000_000.0);
} }
} }
@@ -55,40 +55,30 @@ public class DiskStorage implements AutoCloseable {
fileChannel.close(); fileChannel.close();
} }
public long getNumBlocks() throws IOException { public long[] allocateBlocks(final int numNewBlocks, final int blockSize) throws IOException {
return fileChannel.size() / BLOCK_SIZE;
}
public long[] appendNewBlocks(final int numNewBlocks) throws IOException {
final long[] result = new long[numNewBlocks]; final long[] result = new long[numNewBlocks];
synchronized (fileChannel) { synchronized (fileChannel) {
for (int i = 0; i < numNewBlocks; i++) { for (int i = 0; i < numNewBlocks; i++) {
final long blockNumber = appendNewBlock(); final long blockOffset = allocateBlock(blockSize);
result[i] = blockNumber; result[i] = blockOffset;
} }
} }
return result; return result;
} }
public long appendNewBlock() throws IOException { public long allocateBlock(final int blockSize) throws IOException {
final byte[] buffer = new byte[BLOCK_SIZE]; final byte[] buffer = new byte[blockSize];
final ByteBuffer src = ByteBuffer.wrap(buffer); final ByteBuffer src = ByteBuffer.wrap(buffer);
synchronized (fileChannel) { synchronized (fileChannel) {
// block numbers start with 1, so that the uninitialized value // block numbers start with 1, so that the uninitialized value
// (0) means 'no block'. That way we do not have to write // (0) means 'no block'. That way we do not have to write
// data to a newly created block, which reduces IO. // data to a newly created block, which reduces IO.
final long blockNumber = getNumBlocks() + 1; final long blockOffset = fileChannel.size();
fileChannel.write(src, fileChannel.size()); fileChannel.write(src, fileChannel.size());
return blockNumber; return blockOffset;
} }
} }
public DiskBlock getNewBlock() throws IOException {
final long blockNumber = appendNewBlock();
return getDiskBlock(blockNumber);
}
} }

12
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -41,7 +41,7 @@ public class BSFileTest {
public void testBlockStorage() throws Exception { public void testBlockStorage() throws Exception {
final Path file = dataDirectory.resolve("data.int.db"); final Path file = dataDirectory.resolve("data.int.db");
final int numLongs = 1000; final int numLongs = 1000;
long blockNumber = -1; long blockOffset = -1;
long start = System.nanoTime(); long start = System.nanoTime();
// //
@@ -49,13 +49,13 @@ public class BSFileTest {
try (final BSFile bsFile = BSFile.newFile(ds)) { try (final BSFile bsFile = BSFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockNumber(); blockOffset = bsFile.getRootBlockOffset();
for (long i = 0; i < numLongs / 2; i++) { for (long i = 0; i < numLongs / 2; i++) {
bsFile.append(i); bsFile.append(i);
} }
} }
try (final BSFile bsFile = BSFile.existingFile(blockNumber, ds)) { try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds)) {
for (long i = numLongs / 2; i < numLongs; i++) { for (long i = numLongs / 2; i < numLongs; i++) {
bsFile.append(i); bsFile.append(i);
@@ -66,7 +66,7 @@ public class BSFileTest {
start = System.nanoTime(); start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) { try (final DiskStorage ds = new DiskStorage(file)) {
final BSFile bsFile = BSFile.existingFile(blockNumber, ds); final BSFile bsFile = BSFile.existingFile(blockOffset, ds);
final LongList actualLongs = bsFile.asLongList(); final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1); final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs); Assert.assertEquals(actualLongs, expectedLongs);
@@ -100,7 +100,7 @@ public class BSFileTest {
listOfValues.add(value); listOfValues.add(value);
bsFile.append(value); bsFile.append(value);
} }
expected.put(bsFile.getRootBlockNumber(), listOfValues); expected.put(bsFile.getRootBlockOffset(), listOfValues);
} }
return null; return null;
@@ -146,7 +146,7 @@ public class BSFileTest {
try (final BSFile bsFile = BSFile.newFile(ds)) { try (final BSFile bsFile = BSFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockNumber(); blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) { for (long i = 0; i < numTimeValuePairs / 2; i++) {

26
block-storage/src/test/java/org/lucares/pdb/diskstorage/DiskStorageTest.java
View File

@@ -11,13 +11,13 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.lucares.utils.file.FileUtils; import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod; import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@Test @Test
public class DiskStorageTest { public class DiskStorageTest {
private static final int BLOCK_SIZE = 512;
private Path dataDirectory; private Path dataDirectory;
@@ -45,14 +45,13 @@ public class DiskStorageTest {
try (DiskStorage ds = new DiskStorage(databaseFile)) { try (DiskStorage ds = new DiskStorage(databaseFile)) {
final int numBlocks = 10; final int numBlocks = 10;
ds.appendNewBlocks(numBlocks); ds.allocateBlocks(numBlocks, BLOCK_SIZE);
Assert.assertEquals(ds.getNumBlocks(), numBlocks);
final List<DiskBlock> blocks = new ArrayList<>(); final List<DiskBlock> blocks = new ArrayList<>();
// fill the first 16 512-byte blocks // fill the first 16 512-byte blocks
// that is more than on 4096 byte block // that is more than on 4096 byte block
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i); final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock); assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
diskBlock.writeAsync(); diskBlock.writeAsync();
@@ -70,7 +69,7 @@ public class DiskStorageTest {
// 1. we do this with the existing file channel // 1. we do this with the existing file channel
// this one should see every change, because we wrote them to the file channel // this one should see every change, because we wrote them to the file channel
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i); final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i); assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
blocks.add(diskBlock); blocks.add(diskBlock);
@@ -83,7 +82,7 @@ public class DiskStorageTest {
// use the same buffers from the operating system. // use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile)) { try (DiskStorage ds2 = new DiskStorage(databaseFile)) {
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i); final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i); assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
blocks.add(diskBlock); blocks.add(diskBlock);
@@ -92,7 +91,7 @@ public class DiskStorageTest {
} }
} }
@Test(enabled = false) @Test(enabled = true)
public void testDiskStorage() throws Exception { public void testDiskStorage() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
@@ -101,18 +100,17 @@ public class DiskStorageTest {
try (DiskStorage ds = new DiskStorage(databaseFile)) { try (DiskStorage ds = new DiskStorage(databaseFile)) {
final int numBlocks = 10; final int numBlocks = 10;
ds.appendNewBlocks(numBlocks); final long[] blockOffsets = ds.allocateBlocks(numBlocks, BLOCK_SIZE);
Assert.assertEquals(ds.getNumBlocks(), numBlocks);
for (int i = 0; i < numBlocks; i++) { for (final long blockOffset : blockOffsets) {
final int block = i; final long block = blockOffset;
pool.submit(() -> { pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current(); final ThreadLocalRandom random = ThreadLocalRandom.current();
try { try {
// now read/write random blocks // now read/write random blocks
for (int j = 0; j < 10; j++) { for (int j = 0; j < 10; j++) {
final DiskBlock diskBlock = ds.getDiskBlock(block); final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock); assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) random.nextInt(127)); fill(diskBlock, (byte) random.nextInt(127));
@@ -142,7 +140,7 @@ public class DiskStorageTest {
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
if (expectedVal != buffer[i]) { if (expectedVal != buffer[i]) {
System.err.println( System.err.println(
"block " + diskBlock.getBlockNumber() + " " + buffer[i] + " != " + expectedVal + " at " + i); "block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i);
break; break;
} }
} }
@@ -155,7 +153,7 @@ public class DiskStorageTest {
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
if (expected != buffer[i]) { if (expected != buffer[i]) {
System.err.println( System.err.println(
"block " + diskBlock.getBlockNumber() + " " + buffer[i] + " != " + expected + " at " + i); "block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i);
break; break;
} }
} }

9
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -22,6 +22,7 @@ import java.util.stream.StreamSupport;
import org.lucares.collections.IntList; import org.lucares.collections.IntList;
import org.lucares.pdb.api.StringCompressor; import org.lucares.pdb.api.StringCompressor;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.Doc; import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.lang.Expression; import org.lucares.pdb.datastore.lang.Expression;
@@ -171,13 +172,13 @@ public class DataStore implements AutoCloseable {
public long createNewFile(final Tags tags) throws IOException { public long createNewFile(final Tags tags) throws IOException {
final String filename = tags.serialize(); final String filename = tags.serialize();
final long newFilesRootBlockNumber = diskStorage.appendNewBlock(); final long newFilesRootBlockOffset = diskStorage.allocateBlock(BSFile.BLOCK_SIZE);
updateListingFile(tags, newFilesRootBlockNumber); updateListingFile(tags, newFilesRootBlockOffset);
final ListingFileEntry listingFileEntry = new ListingFileEntry(filename, newFilesRootBlockNumber); final ListingFileEntry listingFileEntry = new ListingFileEntry(filename, newFilesRootBlockOffset);
cacheTagToFileMapping(tags, listingFileEntry); cacheTagToFileMapping(tags, listingFileEntry);
return newFilesRootBlockNumber; return newFilesRootBlockOffset;
} }
private Tags toTags(final String filename) { private Tags toTags(final String filename) {