andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

apply new code formatter and save action

Browse Source
This commit is contained in:
Andreas Huber
2019-11-24 10:20:43 +01:00
parent 5ea82c6a4c
commit 06b379494f
184 changed files with 13455 additions and 13489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFile.java
View File

@@ -37,212 +37,213 @@ import org.slf4j.LoggerFactory;
*/
public class BSFile implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(BSFile.class);
private static final Logger LOGGER = LoggerFactory.getLogger(BSFile.class);
public static final int BLOCK_SIZE = 512;
public static final int BLOCK_SIZE = 512;
/*
* The last disk block of this sequence. This is the block new values will be
* appended to.
*/
private BSFileDiskBlock buffer;
/*
* The last disk block of this sequence. This is the block new values will be
* appended to.
*/
private BSFileDiskBlock buffer;
private int offsetInBuffer = 0;
private int offsetInBuffer = 0;
private boolean dirty = false;
private boolean dirty = false;
private final long rootBlockOffset;
private final long rootBlockOffset;
private final DiskStorage diskStorage;
private final DiskStorage diskStorage;
private final BSFileDiskBlock rootDiskBlock;
private final BSFileDiskBlock rootDiskBlock;
private final BSFileCustomizer customizer;
private final BSFileCustomizer customizer;
BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this(new BSFileDiskBlock(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE)), diskStorage, customizer);
}
this(new BSFileDiskBlock(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE)), diskStorage, customizer);
}
BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this.rootDiskBlock = rootDiskBlock;
this.customizer = customizer;
this.rootBlockOffset = rootDiskBlock.getBlockOffset();
this.diskStorage = diskStorage;
this.rootDiskBlock = rootDiskBlock;
this.customizer = customizer;
this.rootBlockOffset = rootDiskBlock.getBlockOffset();
this.diskStorage = diskStorage;
final long lastBlockNumber = rootDiskBlock.getLastBlockPointer();
if (lastBlockNumber == rootBlockOffset || lastBlockNumber == 0) {
buffer = rootDiskBlock;
} else {
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(lastBlockNumber, BLOCK_SIZE));
}
offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer);
customizer.init(buffer);
LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockOffset, lastBlockNumber);
}
final long lastBlockNumber = rootDiskBlock.getLastBlockPointer();
if (lastBlockNumber == rootBlockOffset || lastBlockNumber == 0) {
buffer = rootDiskBlock;
} else {
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(lastBlockNumber, BLOCK_SIZE));
}
offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer);
customizer.init(buffer);
LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockOffset, lastBlockNumber);
}
private int determineWriteOffsetInExistingBuffer(final BSFileDiskBlock buffer) {
private int determineWriteOffsetInExistingBuffer(final BSFileDiskBlock buffer) {
final byte[] buf = buffer.getBuffer();
final byte[] buf = buffer.getBuffer();
int result = 0;
while (result < buf.length && buf[result] != 0) {
result++;
}
int result = 0;
while (result < buf.length && buf[result] != 0) {
result++;
}
return result;
}
return result;
}
public static BSFile existingFile(final long blockNumber, final DiskStorage diskStorage,
final BSFileCustomizer customizer) {
return new BSFile(blockNumber, diskStorage, customizer);
}
public static BSFile existingFile(final long blockNumber, final DiskStorage diskStorage,
final BSFileCustomizer customizer) {
return new BSFile(blockNumber, diskStorage, customizer);
}
public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) {
final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
LOGGER.trace("create new bsFile={}", rootBlockOffset);
return new BSFile(rootBlockOffset, diskStorage, customizer);
}
public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) {
final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
LOGGER.trace("create new bsFile={}", rootBlockOffset);
return new BSFile(rootBlockOffset, diskStorage, customizer);
}
public void append(final long value1, final long value2) {
final long val1 = customizer.preProcessWriteValue1(value1);
final long val2 = customizer.preProcessWriteValue2(value2);
public void append(final long value1, final long value2) {
final long val1 = customizer.preProcessWriteValue1(value1);
final long val2 = customizer.preProcessWriteValue2(value2);
final int bytesWritten = VariableByteEncoder.encodeInto(val1, val2, buffer.getBuffer(), offsetInBuffer);
final int bytesWritten = VariableByteEncoder.encodeInto(val1, val2, buffer.getBuffer(), offsetInBuffer);
if (bytesWritten == 0) {
flushFullBufferAndCreateNew();
customizer.newBlock();
if (bytesWritten == 0) {
flushFullBufferAndCreateNew();
customizer.newBlock();
append(value1, value2);
}
offsetInBuffer += bytesWritten;
dirty = true;
}
append(value1, value2);
}
offsetInBuffer += bytesWritten;
dirty = true;
}
public void append(final long value) {
int bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
public void append(final long value) {
int bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
if (bytesWritten == 0) {
flushFullBufferAndCreateNew();
bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
assert bytesWritten > 0 : "after a flush the buffer is emtpy, so it should be possible to write a few bytes";
}
offsetInBuffer += bytesWritten;
dirty = true;
}
if (bytesWritten == 0) {
flushFullBufferAndCreateNew();
bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
assert bytesWritten > 0 : "after a flush the buffer is emtpy, so it should be possible to write a few bytes";
}
offsetInBuffer += bytesWritten;
dirty = true;
}
private void flushFullBufferAndCreateNew() {
private void flushFullBufferAndCreateNew() {
final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
if (buffer == rootDiskBlock) {
// root block and current block are the same, so we need
// to update only one
buffer.setLastBlockOffset(newBlockOffset);
buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync();
} else {
rootDiskBlock.writeLastBlockOffset(newBlockOffset);
if (buffer == rootDiskBlock) {
// root block and current block are the same, so we need
// to update only one
buffer.setLastBlockOffset(newBlockOffset);
buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync();
} else {
rootDiskBlock.writeLastBlockOffset(newBlockOffset);
buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync();
}
buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync();
}
// set the new buffer
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(newBlockOffset, BLOCK_SIZE));
offsetInBuffer = 0;
dirty = false;
LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}", rootBlockOffset, newBlockOffset);
}
// set the new buffer
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(newBlockOffset, BLOCK_SIZE));
offsetInBuffer = 0;
dirty = false;
LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}", rootBlockOffset, newBlockOffset);
}
public void flush() {
public void flush() {
LOGGER.trace("flush bsFile={} dirty={} file={}", rootBlockOffset, dirty, diskStorage.getRelativeDatabaseFileForLogging());
if (dirty) {
buffer.writeAsync();
}
}
LOGGER.trace("flush bsFile={} dirty={} file={}", rootBlockOffset, dirty,
diskStorage.getRelativeDatabaseFileForLogging());
if (dirty) {
buffer.writeAsync();
}
}
public Optional<Long> getLastValue() {
public Optional<Long> getLastValue() {
final byte[] buf = buffer.getBuffer();
final LongList bufferedLongs = VariableByteEncoder.decode(buf);
final byte[] buf = buffer.getBuffer();
final LongList bufferedLongs = VariableByteEncoder.decode(buf);
final Optional<Long> result;
if (bufferedLongs.isEmpty()) {
result = Optional.empty();
} else {
final long lastValue = bufferedLongs.get(bufferedLongs.size() - 1);
result = Optional.of(lastValue);
}
return result;
}
final Optional<Long> result;
if (bufferedLongs.isEmpty()) {
result = Optional.empty();
} else {
final long lastValue = bufferedLongs.get(bufferedLongs.size() - 1);
result = Optional.of(lastValue);
}
return result;
}
public Stream<LongList> streamOfLongLists() {
final Iterator<LongList> iterator = new LongListIterator(rootBlockOffset, diskStorage);
final Stream<LongList> stream = StreamSupport
.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
public Stream<LongList> streamOfLongLists() {
final Iterator<LongList> iterator = new LongListIterator(rootBlockOffset, diskStorage);
final Stream<LongList> stream = StreamSupport
.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
final Optional<Function<LongList, LongList>> mapper = customizer.getStreamMapper();
if (mapper.isPresent()) {
return stream.map(mapper.get());
}
return stream;
}
final Optional<Function<LongList, LongList>> mapper = customizer.getStreamMapper();
if (mapper.isPresent()) {
return stream.map(mapper.get());
}
return stream;
}
private static class LongListIterator implements Iterator<LongList> {
private static class LongListIterator implements Iterator<LongList> {
private LongList next = null;
private long nextBlockOffset;
private LongList next = null;
private long nextBlockOffset;
private final DiskStorage diskStorage;
private final DiskStorage diskStorage;
public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) {
this.nextBlockOffset = nextBlockNumber;
this.diskStorage = diskStorage;
}
public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) {
this.nextBlockOffset = nextBlockNumber;
this.diskStorage = diskStorage;
}
@Override
public boolean hasNext() {
return nextBlockOffset != BSFileDiskBlock.NO_NEXT_POINTER;
}
@Override
public boolean hasNext() {
return nextBlockOffset != BSFileDiskBlock.NO_NEXT_POINTER;
}
@Override
public LongList next() {
if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException();
}
@Override
public LongList next() {
if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException();
}
final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset);
nextBlockOffset = diskBlock.getNextBlockNumber();
final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset);
nextBlockOffset = diskBlock.getNextBlockNumber();
final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf);
return next;
}
final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf);
return next;
}
private BSFileDiskBlock getDiskBlock(final long blockOffset) {
final DiskBlock diskBlock = diskStorage.getDiskBlock(blockOffset, BLOCK_SIZE);
return new BSFileDiskBlock(diskBlock);
}
}
private BSFileDiskBlock getDiskBlock(final long blockOffset) {
final DiskBlock diskBlock = diskStorage.getDiskBlock(blockOffset, BLOCK_SIZE);
return new BSFileDiskBlock(diskBlock);
}
}
public LongList asLongList() {
public LongList asLongList() {
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
public long getRootBlockOffset() {
public long getRootBlockOffset() {
return rootBlockOffset;
}
return rootBlockOffset;
}
@Override
public void close() {
flush();
}
@Override
public void close() {
flush();
}
}

10
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFileCustomizer.java
View File

@@ -6,13 +6,13 @@ import java.util.function.Function;
import org.lucares.collections.LongList;
public interface BSFileCustomizer {
void init(BSFileDiskBlock lastDiskBlockOfStream);
void init(BSFileDiskBlock lastDiskBlockOfStream);
Optional<Function<LongList, LongList>> getStreamMapper();
Optional<Function<LongList, LongList>> getStreamMapper();
void newBlock();
void newBlock();
long preProcessWriteValue1(long value);
long preProcessWriteValue1(long value);
long preProcessWriteValue2(long value);
long preProcessWriteValue2(long value);
}

134
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFileDiskBlock.java
View File

@@ -8,90 +8,90 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
class BSFileDiskBlock {
protected static final int NEXT_POINTER_OFFSET = 0;
public static final long NO_NEXT_POINTER = 0;
private static final int LAST_BLOCK_POINTER_POSITION = 8;
public static final long NO_LAST_BLOCK = 0;
private static final int INT_SEQUENCE_OFFSET = 8 // next block pointer
+ 8; // last block pointer;
protected static final int NEXT_POINTER_OFFSET = 0;
public static final long NO_NEXT_POINTER = 0;
private static final int LAST_BLOCK_POINTER_POSITION = 8;
public static final long NO_LAST_BLOCK = 0;
private static final int INT_SEQUENCE_OFFSET = 8 // next block pointer
+ 8; // last block pointer;
private final DiskBlock diskBlock;
private long nextBlockOffset = 0;
private long lastBlockOffset = 0;
private final DiskBlock diskBlock;
private long nextBlockOffset = 0;
private long lastBlockOffset = 0;
private byte[] buffer = null;
private byte[] buffer = null;
public BSFileDiskBlock(final DiskBlock diskBlock) {
this.diskBlock = diskBlock;
}
public BSFileDiskBlock(final DiskBlock diskBlock) {
this.diskBlock = diskBlock;
}
public byte[] getBuffer() {
public byte[] getBuffer() {
if (buffer == null) {
final ByteBuffer byteBuffer = diskBlock.getByteBuffer();
this.buffer = new byte[byteBuffer.capacity() - INT_SEQUENCE_OFFSET];
byteBuffer.position(INT_SEQUENCE_OFFSET);
byteBuffer.get(buffer);
}
if (buffer == null) {
final ByteBuffer byteBuffer = diskBlock.getByteBuffer();
this.buffer = new byte[byteBuffer.capacity() - INT_SEQUENCE_OFFSET];
byteBuffer.position(INT_SEQUENCE_OFFSET);
byteBuffer.get(buffer);
}
return buffer;
}
return buffer;
}
public long getBlockOffset() {
return diskBlock.getBlockOffset();
}
public long getBlockOffset() {
return diskBlock.getBlockOffset();
}
public void setNextBlockOffset(final long nextBlockOffset) {
this.nextBlockOffset = nextBlockOffset;
}
public void setNextBlockOffset(final long nextBlockOffset) {
this.nextBlockOffset = nextBlockOffset;
}
public long getLastBlockPointer() {
public long getLastBlockPointer() {
if (lastBlockOffset <= 0) {
lastBlockOffset = diskBlock.getByteBuffer().getLong(LAST_BLOCK_POINTER_POSITION);
}
if (lastBlockOffset <= 0) {
lastBlockOffset = diskBlock.getByteBuffer().getLong(LAST_BLOCK_POINTER_POSITION);
}
return lastBlockOffset;
}
return lastBlockOffset;
}
public long getNextBlockNumber() {
if (nextBlockOffset <= 0) {
nextBlockOffset = diskBlock.getByteBuffer().getLong(NEXT_POINTER_OFFSET);
}
return nextBlockOffset;
}
public long getNextBlockNumber() {
if (nextBlockOffset <= 0) {
nextBlockOffset = diskBlock.getByteBuffer().getLong(NEXT_POINTER_OFFSET);
}
return nextBlockOffset;
}
public void setLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset;
}
public void setLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset;
}
public void writeLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset;
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
}
public void writeLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset;
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
}
private void writeBufferToByteBuffer() {
diskBlock.getByteBuffer().position(INT_SEQUENCE_OFFSET);
diskBlock.getByteBuffer().put(buffer);
}
private void writeBufferToByteBuffer() {
diskBlock.getByteBuffer().position(INT_SEQUENCE_OFFSET);
diskBlock.getByteBuffer().put(buffer);
}
private void writeBlockHeader() {
diskBlock.getByteBuffer().putLong(NEXT_POINTER_OFFSET, nextBlockOffset);
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
}
private void writeBlockHeader() {
diskBlock.getByteBuffer().putLong(NEXT_POINTER_OFFSET, nextBlockOffset);
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
}
public void writeAsync() {
writeBlockHeader();
writeBufferToByteBuffer();
}
public void writeAsync() {
writeBlockHeader();
writeBufferToByteBuffer();
}
public void force() {
diskBlock.force();
}
public void force() {
diskBlock.force();
}
@Override
public String toString() {
final LongList bufferDecoded = VariableByteEncoder.decode(buffer);
return "BSFileDiskBlock[bufferDecoded=" + bufferDecoded + "]";
}
@Override
public String toString() {
final LongList bufferDecoded = VariableByteEncoder.decode(buffer);
return "BSFileDiskBlock[bufferDecoded=" + bufferDecoded + "]";
}
}

56
block-storage/src/main/java/org/lucares/pdb/blockstorage/LongStreamFile.java
View File

@@ -8,41 +8,41 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class LongStreamFile implements AutoCloseable {
private final BSFile bsFile;
private final BSFile bsFile;
LongStreamFile(final BSFile bsFile) {
this.bsFile = bsFile;
}
LongStreamFile(final BSFile bsFile) {
this.bsFile = bsFile;
}
public static LongStreamFile existingFile(final long blockNumber, final DiskStorage diskStorage)
throws IOException {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile);
}
public static LongStreamFile existingFile(final long blockNumber, final DiskStorage diskStorage)
throws IOException {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile);
}
public static LongStreamFile newFile(final DiskStorage diskStorage) throws IOException {
final BSFile bsFile = BSFile.newFile(diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile);
}
public static LongStreamFile newFile(final DiskStorage diskStorage) throws IOException {
final BSFile bsFile = BSFile.newFile(diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile);
}
public void append(final long value) throws IOException {
public void append(final long value) throws IOException {
bsFile.append(value);
}
bsFile.append(value);
}
public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists();
}
public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists();
}
public LongList asLongList() {
public LongList asLongList() {
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
@Override
public void close() {
bsFile.close();
}
@Override
public void close() {
bsFile.close();
}
}

44
block-storage/src/main/java/org/lucares/pdb/blockstorage/NullCustomizer.java
View File

@@ -7,31 +7,31 @@ import org.lucares.collections.LongList;
public class NullCustomizer implements BSFileCustomizer {
public static final NullCustomizer INSTANCE = new NullCustomizer();
public static final NullCustomizer INSTANCE = new NullCustomizer();
@Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
// nothing to do - this is a NullObject
}
@Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
// nothing to do - this is a NullObject
}
@Override
public Optional<Function<LongList, LongList>> getStreamMapper() {
// no mapper to return - this is a NullObject
return Optional.empty();
}
@Override
public Optional<Function<LongList, LongList>> getStreamMapper() {
// no mapper to return - this is a NullObject
return Optional.empty();
}
@Override
public void newBlock() {
// nothing to do - this is a NullObject
}
@Override
public void newBlock() {
// nothing to do - this is a NullObject
}
@Override
public long preProcessWriteValue1(final long value) {
return value;
}
@Override
public long preProcessWriteValue1(final long value) {
return value;
}
@Override
public long preProcessWriteValue2(final long value) {
return value;
}
@Override
public long preProcessWriteValue2(final long value) {
return value;
}
}

110
block-storage/src/main/java/org/lucares/pdb/blockstorage/TimeSeriesCustomizer.java
View File

@@ -8,71 +8,71 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
public class TimeSeriesCustomizer implements BSFileCustomizer {
private static class TimeStampDeltaDecoder implements Function<LongList, LongList> {
private static class TimeStampDeltaDecoder implements Function<LongList, LongList> {
/**
* Computes the inverse of the delta encoding in {@link BSFile#appendTimeValue}
*/
@Override
public LongList apply(final LongList t) {
long lastTimeValue = 0;
for (int i = 0; i < t.size(); i += 2) {
lastTimeValue += t.get(i);
t.set(i, lastTimeValue);
}
/**
* Computes the inverse of the delta encoding in {@link BSFile#appendTimeValue}
*/
@Override
public LongList apply(final LongList t) {
long lastTimeValue = 0;
for (int i = 0; i < t.size(); i += 2) {
lastTimeValue += t.get(i);
t.set(i, lastTimeValue);
}
return t;
}
}
return t;
}
}
private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder();
private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder();
private long lastEpochMilli;
private long lastEpochMilli;
@Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
lastEpochMilli = determineLastEpochMilli(lastDiskBlockOfStream);
}
@Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
lastEpochMilli = determineLastEpochMilli(lastDiskBlockOfStream);
}
private long determineLastEpochMilli(final BSFileDiskBlock diskBlock) {
private long determineLastEpochMilli(final BSFileDiskBlock diskBlock) {
// get the time/value delta encoded longs
final byte[] buf = diskBlock.getBuffer();
LongList longList = VariableByteEncoder.decode(buf);
final long result;
if (longList.size() < 2) {
// only new files have empty disk blocks
// and empty disk blocks have time offset 0
result = 0;
} else {
// decode the deltas to get the correct timestamps
longList = TIME_DELTA_DECODER.apply(longList);
// get the time/value delta encoded longs
final byte[] buf = diskBlock.getBuffer();
LongList longList = VariableByteEncoder.decode(buf);
final long result;
if (longList.size() < 2) {
// only new files have empty disk blocks
// and empty disk blocks have time offset 0
result = 0;
} else {
// decode the deltas to get the correct timestamps
longList = TIME_DELTA_DECODER.apply(longList);
// return the last timestamp
result = longList.get(longList.size() - 2);
}
return result;
}
// return the last timestamp
result = longList.get(longList.size() - 2);
}
return result;
}
@Override
public Optional<Function<LongList, LongList>> getStreamMapper() {
return Optional.of(TIME_DELTA_DECODER);
}
@Override
public Optional<Function<LongList, LongList>> getStreamMapper() {
return Optional.of(TIME_DELTA_DECODER);
}
@Override
public void newBlock() {
lastEpochMilli = 0;
}
@Override
public void newBlock() {
lastEpochMilli = 0;
}
@Override
public long preProcessWriteValue1(final long epochMilli) {
final long epochMilliDelta = epochMilli - lastEpochMilli;
lastEpochMilli = epochMilli;
return epochMilliDelta;
}
@Override
public long preProcessWriteValue1(final long epochMilli) {
final long epochMilliDelta = epochMilli - lastEpochMilli;
lastEpochMilli = epochMilli;
return epochMilliDelta;
}
@Override
public long preProcessWriteValue2(final long value) {
return value;
}
@Override
public long preProcessWriteValue2(final long value) {
return value;
}
}

72
block-storage/src/main/java/org/lucares/pdb/blockstorage/TimeSeriesFile.java
View File

@@ -8,52 +8,52 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class TimeSeriesFile implements AutoCloseable {
private final BSFile bsFile;
private final BSFile bsFile;
private TimeSeriesFile(final BSFile bsFile) {
this.bsFile = bsFile;
}
private TimeSeriesFile(final BSFile bsFile) {
this.bsFile = bsFile;
}
public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public static TimeSeriesFile newFile(final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.newFile(diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public static TimeSeriesFile newFile(final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.newFile(diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile);
}
public void appendTimeValue(final long epochMilli, final long value) {
public void appendTimeValue(final long epochMilli, final long value) {
bsFile.append(epochMilli, value);
}
bsFile.append(epochMilli, value);
}
public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists();
}
public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists();
}
public LongList asTimeValueLongList() {
public LongList asTimeValueLongList() {
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll);
return result;
}
@Override
public void close() {
bsFile.close();
}
@Override
public void close() {
bsFile.close();
}
public long getRootBlockOffset() {
return bsFile.getRootBlockOffset();
}
public long getRootBlockOffset() {
return bsFile.getRootBlockOffset();
}
public Optional<Long> getLastValue() {
return bsFile.getLastValue();
}
public Optional<Long> getLastValue() {
return bsFile.getLastValue();
}
public void flush() {
bsFile.flush();
}
public void flush() {
bsFile.flush();
}
}

74
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskBlock.java
View File

@@ -5,52 +5,52 @@ import java.nio.MappedByteBuffer;
public class DiskBlock {
private byte[] buffer = null;
private final long blockOffset;
private byte[] buffer = null;
private final long blockOffset;
private final ByteBuffer byteBuffer;
private final ByteBuffer byteBuffer;
public DiskBlock(final long blockOffset, final ByteBuffer byteBuffer) {
this.blockOffset = blockOffset;
this.byteBuffer = byteBuffer;
}
public DiskBlock(final long blockOffset, final ByteBuffer byteBuffer) {
this.blockOffset = blockOffset;
this.byteBuffer = byteBuffer;
}
public byte[] getBuffer() {
public byte[] getBuffer() {
if (buffer == null) {
this.buffer = new byte[byteBuffer.capacity()];
byteBuffer.get(buffer);
}
if (buffer == null) {
this.buffer = new byte[byteBuffer.capacity()];
byteBuffer.get(buffer);
}
return buffer;
}
return buffer;
}
public ByteBuffer getByteBuffer() {
return byteBuffer;
}
public ByteBuffer getByteBuffer() {
return byteBuffer;
}
public long getBlockOffset() {
return blockOffset;
}
public long getBlockOffset() {
return blockOffset;
}
private void writeBufferToByteBuffer() {
byteBuffer.position(0);
byteBuffer.put(buffer);
}
private void writeBufferToByteBuffer() {
byteBuffer.position(0);
byteBuffer.put(buffer);
}
public void writeAsync() {
writeBufferToByteBuffer();
}
public void writeAsync() {
writeBufferToByteBuffer();
}
public void force() {
// some tests use HeapByteBuffer and don't support force
if (byteBuffer instanceof MappedByteBuffer) {
((MappedByteBuffer) byteBuffer).force();
}
}
public void force() {
// some tests use HeapByteBuffer and don't support force
if (byteBuffer instanceof MappedByteBuffer) {
((MappedByteBuffer) byteBuffer).force();
}
}
@Override
public String toString() {
return "DiskBlock[" + blockOffset + "]";
}
@Override
public String toString() {
return "DiskBlock[" + blockOffset + "]";
}
}

438
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorage.java
View File

@@ -14,273 +14,273 @@ import org.slf4j.LoggerFactory;
public class DiskStorage implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class);
private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class);
private static final long FREE_LIST_ROOT_OFFSET = 0;
private static final long NO_POINTER = 0;
private static final int FREE_LIST_NEXT_POINTER = 0;
private static final int FREE_LIST_PREV_POINTER = 8;
private static final int FREE_LIST_SIZE = 16;
private static final int FREE_LIST_NODE_SIZE = 32;
private static final long FREE_LIST_ROOT_OFFSET = 0;
private static final long NO_POINTER = 0;
private static final int FREE_LIST_NEXT_POINTER = 0;
private static final int FREE_LIST_PREV_POINTER = 8;
private static final int FREE_LIST_SIZE = 16;
private static final int FREE_LIST_NODE_SIZE = 32;
private final FileChannel fileChannel;
private final FileChannel fileChannel;
private Path relativeDatabaseFileForLogging;
private Path relativeDatabaseFileForLogging;
public DiskStorage(final Path databaseFile, Path storageBasePath) {
this.relativeDatabaseFileForLogging = storageBasePath != null ? storageBasePath.relativize(databaseFile): databaseFile;
try {
Files.createDirectories(databaseFile.getParent());
public DiskStorage(final Path databaseFile, Path storageBasePath) {
this.relativeDatabaseFileForLogging = storageBasePath != null ? storageBasePath.relativize(databaseFile)
: databaseFile;
try {
Files.createDirectories(databaseFile.getParent());
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE);
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE);
initIfNew();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
initIfNew();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
private void initIfNew() throws IOException {
if (fileChannel.size() == 0) {
// file is new -> add root of the free list
writeFreeListRootNodePosition(NO_POINTER);
}
}
private void initIfNew() throws IOException {
if (fileChannel.size() == 0) {
// file is new -> add root of the free list
writeFreeListRootNodePosition(NO_POINTER);
}
}
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) {
try {
LOGGER.trace("read block={} file={}", blockOffset, relativeDatabaseFileForLogging);
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) {
try {
LOGGER.trace("read block={} file={}", blockOffset, relativeDatabaseFileForLogging);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
return new DiskBlock(blockOffset, byteBuffer);
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public Path getRelativeDatabaseFileForLogging() {
return relativeDatabaseFileForLogging;
}
return new DiskBlock(blockOffset, byteBuffer);
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
@Override
public void close() {
try {
fileChannel.force(true);
fileChannel.close();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public Path getRelativeDatabaseFileForLogging() {
return relativeDatabaseFileForLogging;
}
public synchronized long allocateBlock(final int blockSize) {
if (blockSize < FREE_LIST_NODE_SIZE) {
throw new IllegalArgumentException("The minimal allocation size is 32 byte.");
}
@Override
public void close() {
try {
fileChannel.force(true);
fileChannel.close();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
try {
final var optionalFreeBlock = findFreeBlockWithSize(blockSize);
if (optionalFreeBlock.isPresent()) {
final FreeListNode freeBlock = optionalFreeBlock.get();
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public synchronized long allocateBlock(final int blockSize) {
if (blockSize < FREE_LIST_NODE_SIZE) {
throw new IllegalArgumentException("The minimal allocation size is 32 byte.");
}
private long allocateNewBlock(final int blockSize) throws IOException {
final var buffer = new byte[blockSize];
final var src = ByteBuffer.wrap(buffer);
try {
final var optionalFreeBlock = findFreeBlockWithSize(blockSize);
if (optionalFreeBlock.isPresent()) {
final FreeListNode freeBlock = optionalFreeBlock.get();
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
// block numbers start with 1, so that the uninitialized value
// (0) means 'no block'. That way we do not have to write
// data to a newly created block, which reduces IO.
final var blockOffset = fileChannel.size();
fileChannel.write(src, fileChannel.size());
return blockOffset;
}
private long allocateNewBlock(final int blockSize) throws IOException {
final var buffer = new byte[blockSize];
final var src = ByteBuffer.wrap(buffer);
public synchronized void free(final long blockOffset, final int blockSize) throws IOException {
// block numbers start with 1, so that the uninitialized value
// (0) means 'no block'. That way we do not have to write
// data to a newly created block, which reduces IO.
final var blockOffset = fileChannel.size();
fileChannel.write(src, fileChannel.size());
return blockOffset;
}
final var neighboringFreeListNode = getNeighboringFreeListNode(blockOffset);
public synchronized void free(final long blockOffset, final int blockSize) throws IOException {
if (neighboringFreeListNode.isPresent()) {
// insert new free node into the free list
final var prev = neighboringFreeListNode.get();
final var neighboringFreeListNode = getNeighboringFreeListNode(blockOffset);
insertFreeListNode(prev, blockOffset, blockSize);
if (neighboringFreeListNode.isPresent()) {
// insert new free node into the free list
final var prev = neighboringFreeListNode.get();
} else {
// add new free list node as the first node in the list
insertFreeListNodeAsNewRoot(blockOffset, blockSize);
}
}
insertFreeListNode(prev, blockOffset, blockSize);
private void insertFreeListNodeAsNewRoot(final long blockOffset, final int blockSize) throws IOException {
final var freeListRootNodePosition = readFreeListRootNodePosition();
} else {
// add new free list node as the first node in the list
insertFreeListNodeAsNewRoot(blockOffset, blockSize);
}
}
if (freeListRootNodePosition > 0) {
// there are free list nodes, but they are after the new node
private void insertFreeListNodeAsNewRoot(final long blockOffset, final int blockSize) throws IOException {
final var freeListRootNodePosition = readFreeListRootNodePosition();
final var next = readFreeListNode(freeListRootNodePosition);
final var newNode = new FreeListNode(blockOffset, blockSize);
if (freeListRootNodePosition > 0) {
// there are free list nodes, but they are after the new node
FreeListNode.link(newNode, next);
final var next = readFreeListNode(freeListRootNodePosition);
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode, next);
writeFreeListRootNodePosition(blockOffset);
FreeListNode.link(newNode, next);
} else {
// this is the first free list node
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode);
writeFreeListRootNodePosition(blockOffset);
}
}
writeFreeListNode(newNode, next);
writeFreeListRootNodePosition(blockOffset);
private void insertFreeListNode(final FreeListNode prev, final long blockOffset, final int blockSize)
throws IOException {
} else {
// this is the first free list node
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode);
writeFreeListRootNodePosition(blockOffset);
}
}
final var newNode = new FreeListNode(blockOffset, blockSize);
final var next = prev.hasNext() ? readFreeListNode(prev.getNext()) : null;
private void insertFreeListNode(final FreeListNode prev, final long blockOffset, final int blockSize)
throws IOException {
FreeListNode.link(prev, newNode, next);
final var newNode = new FreeListNode(blockOffset, blockSize);
final var next = prev.hasNext() ? readFreeListNode(prev.getNext()) : null;
writeFreeListNode(prev, newNode, next);
}
FreeListNode.link(prev, newNode, next);
/**
*
* @param blockOffset the offset of the block that is about to be free'd
* @return the free list node before the block
* @throws IOException
*/
private Optional<FreeListNode> getNeighboringFreeListNode(final long blockOffset) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListRootNodePosition < blockOffset) {
writeFreeListNode(prev, newNode, next);
}
long nextFreeListNodeOffset = freeListRootNodePosition;
while (nextFreeListNodeOffset > 0) {
final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
/**
*
* @param blockOffset the offset of the block that is about to be free'd
* @return the free list node before the block
* @throws IOException
*/
private Optional<FreeListNode> getNeighboringFreeListNode(final long blockOffset) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListRootNodePosition < blockOffset) {
if (freeListNode.getOffset() > blockOffset) {
break;
}
nextFreeListNodeOffset = freeListNode.getNext();
result = freeListNode;
}
}
return Optional.ofNullable(result);
}
long nextFreeListNodeOffset = freeListRootNodePosition;
while (nextFreeListNodeOffset > 0) {
final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
private Optional<FreeListNode> findFreeBlockWithSize(final long blockSize) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListNode.getOffset() > blockOffset) {
break;
}
nextFreeListNodeOffset = freeListNode.getNext();
result = freeListNode;
}
}
return Optional.ofNullable(result);
}
long nextFreeListNodeOffset = freeListRootNodePosition;
while (nextFreeListNodeOffset > 0) {
final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
private Optional<FreeListNode> findFreeBlockWithSize(final long blockSize) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListNode.getSize() == blockSize) {
result = freeListNode;
break;
}
nextFreeListNodeOffset = freeListNode.getNext();
}
long nextFreeListNodeOffset = freeListRootNodePosition;
while (nextFreeListNodeOffset > 0) {
final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
return Optional.ofNullable(result);
}
if (freeListNode.getSize() == blockSize) {
result = freeListNode;
break;
}
nextFreeListNodeOffset = freeListNode.getNext();
}
private void clearBlock(final FreeListNode freeBlock) throws IOException {
final var src = ByteBuffer.allocate(freeBlock.getSize());
fileChannel.write(src, freeBlock.getOffset());
}
return Optional.ofNullable(result);
}
private void removeBlockFromFreeList(final FreeListNode freeBlock) throws IOException {
private void clearBlock(final FreeListNode freeBlock) throws IOException {
final var src = ByteBuffer.allocate(freeBlock.getSize());
fileChannel.write(src, freeBlock.getOffset());
}
if (freeBlock.getPrev() == 0) {
writeFreeListRootNodePosition(freeBlock.getNext());
}
private void removeBlockFromFreeList(final FreeListNode freeBlock) throws IOException {
if (freeBlock.getNext() > 0) {
final FreeListNode next = readFreeListNode(freeBlock.getNext());
next.setPrev(freeBlock.getPrev());
writeFreeListNode(next);
}
if (freeBlock.getPrev() == 0) {
writeFreeListRootNodePosition(freeBlock.getNext());
}
if (freeBlock.getPrev() > 0) {
final FreeListNode prev = readFreeListNode(freeBlock.getPrev());
prev.setNext(freeBlock.getNext());
writeFreeListNode(prev);
}
}
if (freeBlock.getNext() > 0) {
final FreeListNode next = readFreeListNode(freeBlock.getNext());
next.setPrev(freeBlock.getPrev());
writeFreeListNode(next);
}
private FreeListNode readFreeListNode(final long freeListNodePosition) throws IOException {
final var freeListNode = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
fileChannel.read(freeListNode, freeListNodePosition);
final long offset = freeListNodePosition;
final long next = freeListNode.getLong(FREE_LIST_NEXT_POINTER);
final long prev = freeListNode.getLong(FREE_LIST_PREV_POINTER);
final int size = freeListNode.getInt(FREE_LIST_SIZE);
return new FreeListNode(offset, next, prev, size);
}
if (freeBlock.getPrev() > 0) {
final FreeListNode prev = readFreeListNode(freeBlock.getPrev());
prev.setNext(freeBlock.getNext());
writeFreeListNode(prev);
}
}
private void writeFreeListNode(final FreeListNode... nodes) throws IOException {
private FreeListNode readFreeListNode(final long freeListNodePosition) throws IOException {
final var freeListNode = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
fileChannel.read(freeListNode, freeListNodePosition);
final long offset = freeListNodePosition;
final long next = freeListNode.getLong(FREE_LIST_NEXT_POINTER);
final long prev = freeListNode.getLong(FREE_LIST_PREV_POINTER);
final int size = freeListNode.getInt(FREE_LIST_SIZE);
return new FreeListNode(offset, next, prev, size);
}
for (final FreeListNode node : nodes) {
if (node != null) {
final var src = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
src.putLong(FREE_LIST_NEXT_POINTER, node.getNext());
src.putLong(FREE_LIST_PREV_POINTER, node.getPrev());
src.putInt(FREE_LIST_SIZE, node.getSize());
fileChannel.write(src, node.getOffset());
}
}
}
private void writeFreeListNode(final FreeListNode... nodes) throws IOException {
private long readFreeListRootNodePosition() throws IOException {
final var freeListFirstBlock = ByteBuffer.allocate(8);
fileChannel.read(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
return freeListFirstBlock.getLong(0);
}
for (final FreeListNode node : nodes) {
if (node != null) {
final var src = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
src.putLong(FREE_LIST_NEXT_POINTER, node.getNext());
src.putLong(FREE_LIST_PREV_POINTER, node.getPrev());
src.putInt(FREE_LIST_SIZE, node.getSize());
fileChannel.write(src, node.getOffset());
}
}
}
private void writeFreeListRootNodePosition(final long freeListRootNodePosition) throws IOException {
final var freeListFirstBlock = ByteBuffer.allocate(8);
freeListFirstBlock.putLong(0, freeListRootNodePosition);
fileChannel.write(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
}
private long readFreeListRootNodePosition() throws IOException {
final var freeListFirstBlock = ByteBuffer.allocate(8);
fileChannel.read(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
return freeListFirstBlock.getLong(0);
}
public synchronized void ensureAlignmentForNewBlocks(final int alignment) {
try {
final long size = fileChannel.size();
final int alignmentMismatch = Math.floorMod(size, alignment);
if (alignmentMismatch != 0) {
// The next allocated block would not be aligned. Therefore we allocate a
// throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
private void writeFreeListRootNodePosition(final long freeListRootNodePosition) throws IOException {
final var freeListFirstBlock = ByteBuffer.allocate(8);
freeListFirstBlock.putLong(0, freeListRootNodePosition);
fileChannel.write(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
}
public long size() {
try {
return fileChannel.size();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public synchronized void ensureAlignmentForNewBlocks(final int alignment) {
try {
final long size = fileChannel.size();
final int alignmentMismatch = Math.floorMod(size, alignment);
if (alignmentMismatch != 0) {
// The next allocated block would not be aligned. Therefore we allocate a
// throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public int minAllocationSize() {
return FREE_LIST_NODE_SIZE;
}
public long size() {
try {
return fileChannel.size();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public int minAllocationSize() {
return FREE_LIST_NODE_SIZE;
}
}

20
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorageException.java
View File

@@ -2,18 +2,18 @@ package org.lucares.pdb.diskstorage;
public class DiskStorageException extends RuntimeException {
private static final long serialVersionUID = 1683775743640383633L;
private static final long serialVersionUID = 1683775743640383633L;
public DiskStorageException(final String message, final Throwable cause) {
super(message, cause);
}
public DiskStorageException(final String message, final Throwable cause) {
super(message, cause);
}
public DiskStorageException(final String message) {
super(message);
}
public DiskStorageException(final String message) {
super(message);
}
public DiskStorageException(final Throwable cause) {
super(cause);
}
public DiskStorageException(final Throwable cause) {
super(cause);
}
}

124
block-storage/src/main/java/org/lucares/pdb/diskstorage/FreeListNode.java
View File

@@ -1,82 +1,82 @@
package org.lucares.pdb.diskstorage;
public class FreeListNode {
private final long offset;
private long next;
private long prev;
private int size;
private final long offset;
private long next;
private long prev;
private int size;
public FreeListNode(final long offset, final int size) {
this.offset = offset;
this.size = size;
}
public FreeListNode(final long offset, final int size) {
this.offset = offset;
this.size = size;
}
public FreeListNode(final long offset, final long next, final long prev, final int size) {
this.offset = offset;
this.next = next;
this.prev = prev;
this.size = size;
}
public FreeListNode(final long offset, final long next, final long prev, final int size) {
this.offset = offset;
this.next = next;
this.prev = prev;
this.size = size;
}
public long getOffset() {
return offset;
}
public long getOffset() {
return offset;
}
public long getNext() {
return next;
}
public long getNext() {
return next;
}
public void setNext(final long next) {
this.next = next;
}
public void setNext(final long next) {
this.next = next;
}
public void setNext(final FreeListNode next) {
this.next = next != null ? next.getOffset() : 0;
}
public void setNext(final FreeListNode next) {
this.next = next != null ? next.getOffset() : 0;
}
public long getPrev() {
return prev;
}
public long getPrev() {
return prev;
}
public void setPrev(final long prev) {
this.prev = prev;
}
public void setPrev(final long prev) {
this.prev = prev;
}
public void setPrev(final FreeListNode prev) {
this.prev = prev != null ? prev.getOffset() : 0;
}
public void setPrev(final FreeListNode prev) {
this.prev = prev != null ? prev.getOffset() : 0;
}
public int getSize() {
return size;
}
public int getSize() {
return size;
}
public void setSize(final int size) {
this.size = size;
}
public void setSize(final int size) {
this.size = size;
}
@Override
public String toString() {
return "FreeListNode [offset=" + offset + ", next=" + next + ", prev=" + prev + ", size=" + size + "]";
}
@Override
public String toString() {
return "FreeListNode [offset=" + offset + ", next=" + next + ", prev=" + prev + ", size=" + size + "]";
}
public boolean hasNext() {
return next != 0;
}
public boolean hasNext() {
return next != 0;
}
public static void link(final FreeListNode prev, final FreeListNode next) {
prev.setNext(next);
next.setPrev(prev);
}
public static void link(final FreeListNode prev, final FreeListNode next) {
prev.setNext(next);
next.setPrev(prev);
}
public static void link(final FreeListNode prev, final FreeListNode middle, final FreeListNode next) {
if (prev != null) {
prev.setNext(middle);
}
middle.setPrev(prev);
middle.setNext(next);
if (next != null) {
next.setPrev(prev);
}
}
public static void link(final FreeListNode prev, final FreeListNode middle, final FreeListNode next) {
if (prev != null) {
prev.setNext(middle);
}
middle.setPrev(prev);
middle.setNext(next);
if (next != null) {
next.setPrev(prev);
}
}
}

122
block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java
View File

@@ -3,77 +3,77 @@ package org.lucares.pdb.map;
import java.util.Arrays;
public final class ByteArrayKey implements Comparable<ByteArrayKey> {
private final byte[] bytes;
private final byte[] bytes;
public ByteArrayKey(final byte[] bytes) {
this.bytes = bytes;
}
public ByteArrayKey(final byte[] bytes) {
this.bytes = bytes;
}
@Override
public int compareTo(final ByteArrayKey o) {
return compare(bytes, o.bytes);
}
@Override
public int compareTo(final ByteArrayKey o) {
return compare(bytes, o.bytes);
}
public static int compare(final byte[] key, final byte[] otherKey) {
return Arrays.compare(key, otherKey);
}
public static int compare(final byte[] key, final byte[] otherKey) {
return Arrays.compare(key, otherKey);
}
public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) {
public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) {
return compareKeyPrefix(key, keyPrefix) == 0;
}
return compareKeyPrefix(key, keyPrefix) == 0;
}
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public static int compareKeyPrefix(final byte[] key, final byte[] prefix) {
int i = 0;
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public static int compareKeyPrefix(final byte[] key, final byte[] prefix) {
int i = 0;
while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) {
return key[i] - prefix[i];
}
i++;
}
return key.length > prefix.length ? 0 : key.length - prefix.length;
return key.length > prefix.length ? 0 : key.length - prefix.length;
}
}
public static boolean equal(final byte[] key, final byte[] otherKey) {
return compare(key, otherKey) == 0;
}
@Override
public String toString() {
return Arrays.toString(bytes);
}
public static boolean equal(final byte[] key, final byte[] otherKey) {
return compare(key, otherKey) == 0;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(bytes);
return result;
}
@Override
public String toString() {
return Arrays.toString(bytes);
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ByteArrayKey other = (ByteArrayKey) obj;
if (!Arrays.equals(bytes, other.bytes))
return false;
return true;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(bytes);
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ByteArrayKey other = (ByteArrayKey) obj;
if (!Arrays.equals(bytes, other.bytes))
return false;
return true;
}
}

14
block-storage/src/main/java/org/lucares/pdb/map/Empty.java
View File

@@ -14,13 +14,13 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
* {@link Empty} solves this by providing a single unmodifiable value.
*/
public final class Empty {
public static final Empty INSTANCE = new Empty();
public static final Empty INSTANCE = new Empty();
private Empty() {
}
private Empty() {
}
@Override
public String toString() {
return "<empty>";
}
@Override
public String toString() {
return "<empty>";
}
}

244
block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java
View File

@@ -9,158 +9,158 @@ import java.util.function.Predicate;
import org.lucares.utils.byteencoder.VariableByteEncoder;
class NodeEntry {
enum ValueType {
VALUE_INLINE((byte) 1), NODE_POINTER((byte) 2);
enum ValueType {
VALUE_INLINE((byte) 1), NODE_POINTER((byte) 2);
private final byte b;
private final byte b;
ValueType(final byte b) {
this.b = b;
}
ValueType(final byte b) {
this.b = b;
}
static ValueType fromByte(final byte b) {
for (final ValueType type : values()) {
if (type.b == b) {
return type;
}
}
throw new IllegalStateException("Cannot map byte " + b + " to a value type.");
}
static ValueType fromByte(final byte b) {
for (final ValueType type : values()) {
if (type.b == b) {
return type;
}
}
throw new IllegalStateException("Cannot map byte " + b + " to a value type.");
}
public byte asByte() {
return b;
}
}
public byte asByte() {
return b;
}
}
static final class KeyMatches implements Predicate<NodeEntry> {
static final class KeyMatches implements Predicate<NodeEntry> {
private final byte[] key;
private final byte[] key;
public KeyMatches(final byte[] key) {
this.key = key;
}
public KeyMatches(final byte[] key) {
this.key = key;
}
@Override
public boolean test(final NodeEntry t) {
return Arrays.equals(key, t.getKey());
}
}
@Override
public boolean test(final NodeEntry t) {
return Arrays.equals(key, t.getKey());
}
}
private final ValueType type;
private final byte[] key;
private final byte[] value;
private final ValueType type;
private final byte[] key;
private final byte[] value;
public NodeEntry(final ValueType type, final byte[] key, final byte[] value) {
this.type = type;
this.key = key;
this.value = value;
}
public NodeEntry(final ValueType type, final byte[] key, final byte[] value) {
this.type = type;
this.key = key;
this.value = value;
}
public ValueType getType() {
return type;
}
public ValueType getType() {
return type;
}
public byte[] getKey() {
return key;
}
public byte[] getKey() {
return key;
}
public byte[] getValue() {
return value;
}
public byte[] getValue() {
return value;
}
public int size() {
return 1 + key.length + value.length;
}
public int size() {
return 1 + key.length + value.length;
}
@Override
public String toString() {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: new String(value, StandardCharsets.UTF_8);
@Override
public String toString() {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: new String(value, StandardCharsets.UTF_8);
return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value="
+ valueAsString + "]";
}
return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value="
+ valueAsString + "]";
}
public <K,V> String toString(final Function<byte[], K> keyDecoder, final Function<byte[], V> valueDecoder) {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: String.valueOf(valueDecoder.apply(value));
public <K, V> String toString(final Function<byte[], K> keyDecoder, final Function<byte[], V> valueDecoder) {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: String.valueOf(valueDecoder.apply(value));
final String keyAsString;
if (Arrays.equals(key, PersistentMap.MAX_KEY)) {
keyAsString = "<<<MAX_KEY>>>";
} else {
keyAsString = String.valueOf(keyDecoder.apply(key));
}
final String keyAsString;
if (Arrays.equals(key, PersistentMap.MAX_KEY)) {
keyAsString = "<<<MAX_KEY>>>";
} else {
keyAsString = String.valueOf(keyDecoder.apply(key));
}
return "NodeEntry [type=" + type + ", key=" + keyAsString + ", value=" + valueAsString + "]";
}
return "NodeEntry [type=" + type + ", key=" + keyAsString + ", value=" + valueAsString + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(key);
result = prime * result + ((type == null) ? 0 : type.hashCode());
result = prime * result + Arrays.hashCode(value);
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(key);
result = prime * result + ((type == null) ? 0 : type.hashCode());
result = prime * result + Arrays.hashCode(value);
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final NodeEntry other = (NodeEntry) obj;
if (!Arrays.equals(key, other.key))
return false;
if (type != other.type)
return false;
if (!Arrays.equals(value, other.value))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final NodeEntry other = (NodeEntry) obj;
if (!Arrays.equals(key, other.key))
return false;
if (type != other.type)
return false;
if (!Arrays.equals(value, other.value))
return false;
return true;
}
public static int neededBytes(final Collection<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum();
}
public static int neededBytes(final Collection<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum();
}
public int compare(final byte[] otherKey) {
public int compare(final byte[] otherKey) {
return ByteArrayKey.compare(key, otherKey);
}
return ByteArrayKey.compare(key, otherKey);
}
public boolean isPrefix(final byte[] keyPrefix) {
public boolean isPrefix(final byte[] keyPrefix) {
return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0;
}
return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0;
}
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public int compareKeyPrefix(final byte[] prefix) {
/**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix.
*
* @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])}
*/
public int compareKeyPrefix(final byte[] prefix) {
return ByteArrayKey.compareKeyPrefix(key, prefix);
}
return ByteArrayKey.compareKeyPrefix(key, prefix);
}
public boolean equal(final byte[] otherKey) {
return compare(otherKey) == 0;
}
public boolean equal(final byte[] otherKey) {
return compare(otherKey) == 0;
}
public boolean isDataNode() {
return type == ValueType.VALUE_INLINE;
}
public boolean isDataNode() {
return type == ValueType.VALUE_INLINE;
}
public boolean isInnerNode() {
return type == ValueType.NODE_POINTER;
}
public boolean isInnerNode() {
return type == ValueType.NODE_POINTER;
}
}

884
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -23,470 +23,470 @@ import org.slf4j.LoggerFactory;
public class PersistentMap<K, V> implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class);
// the maximum key
static final byte[] MAX_KEY;
static {
MAX_KEY = new byte[20];
Arrays.fill(MAX_KEY, Byte.MAX_VALUE);
}
// the maximum key
static final byte[] MAX_KEY;
static {
MAX_KEY = new byte[20];
Arrays.fill(MAX_KEY, Byte.MAX_VALUE);
}
interface VisitorCallback {
void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth);
}
interface VisitorCallback {
void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth);
}
public interface EncoderDecoder<O> {
public byte[] encode(O object);
public O decode(byte[] bytes);
public interface EncoderDecoder<O> {
public byte[] encode(O object);
public O decode(byte[] bytes);
public default Function<byte[], O> asDecoder() {
return bytes -> this.decode(bytes);
}
public default Function<byte[], O> asDecoder() {
return bytes -> this.decode(bytes);
}
public default Function<O, byte[]> asEncoder() {
return plain -> this.encode(plain);
}
public default Function<O, byte[]> asEncoder() {
return plain -> this.encode(plain);
}
public byte[] getEmptyValue();
}
public byte[] getEmptyValue();
}
private static final class StringCoder implements EncoderDecoder<String> {
private static final class StringCoder implements EncoderDecoder<String> {
@Override
public byte[] encode(final String object) {
return object.getBytes(StandardCharsets.UTF_8);
}
@Override
public byte[] encode(final String object) {
return object.getBytes(StandardCharsets.UTF_8);
}
@Override
public String decode(final byte[] bytes) {
return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8);
}
@Override
public String decode(final byte[] bytes) {
return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private static final class LongCoder implements EncoderDecoder<Long> {
private static final class LongCoder implements EncoderDecoder<Long> {
@Override
public byte[] encode(final Long object) {
return VariableByteEncoder.encode(object);
}
@Override
public byte[] encode(final Long object) {
return VariableByteEncoder.encode(object);
}
@Override
public Long decode(final byte[] bytes) {
return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes);
}
@Override
public Long decode(final byte[] bytes) {
return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private static final class UUIDCoder implements EncoderDecoder<UUID> {
private static final class UUIDCoder implements EncoderDecoder<UUID> {
@Override
public byte[] encode(final UUID uuid) {
final long mostSignificantBits = uuid.getMostSignificantBits();
final long leastSignificantBits = uuid.getLeastSignificantBits();
return VariableByteEncoder.encode(mostSignificantBits, leastSignificantBits);
}
@Override
public byte[] encode(final UUID uuid) {
final long mostSignificantBits = uuid.getMostSignificantBits();
final long leastSignificantBits = uuid.getLeastSignificantBits();
return VariableByteEncoder.encode(mostSignificantBits, leastSignificantBits);
}
@Override
public UUID decode(final byte[] bytes) {
@Override
public UUID decode(final byte[] bytes) {
final LongList longs = VariableByteEncoder.decode(bytes);
final long mostSignificantBits = longs.get(0);
final long leastSignificantBits = longs.get(1);
final LongList longs = VariableByteEncoder.decode(bytes);
final long mostSignificantBits = longs.get(0);
final long leastSignificantBits = longs.get(1);
return new UUID(mostSignificantBits, leastSignificantBits);
}
return new UUID(mostSignificantBits, leastSignificantBits);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private static final class EmptyCoder implements EncoderDecoder<Empty> {
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
private static final class EmptyCoder implements EncoderDecoder<Empty> {
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@Override
public byte[] encode(final Empty __) {
return EMPTY_BYTE_ARRAY;
}
@Override
public byte[] encode(final Empty __) {
return EMPTY_BYTE_ARRAY;
}
@Override
public Empty decode(final byte[] bytes) {
Preconditions.checkTrue(bytes.length == 0, "");
return Empty.INSTANCE;
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
}
public static final EncoderDecoder<Long> LONG_CODER = new LongCoder();
public static final EncoderDecoder<UUID> UUID_ENCODER = new UUIDCoder();
public static final EncoderDecoder<String> STRING_CODER = new StringCoder();
public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder();
static final int BLOCK_SIZE = 4096;
static final long NODE_OFFSET_TO_ROOT_NODE = 8;
private final DiskStorage diskStore;
private int maxEntriesInNode = Integer.MAX_VALUE;
private final EncoderDecoder<K> keyEncoder;
private final EncoderDecoder<V> valueEncoder;
private final LRUCache<Long, PersistentMapDiskNode> nodeCache = new LRUCache<>(10_000);
private final LRUCache<K, V> valueCache = new LRUCache<>(1_000);
public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder,
final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path, storageBasePath);
this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder;
initIfNew();
}
@Override
public void close() {
diskStore.close();
}
public void setMaxEntriesInNode(final int maxEntriesInNode) {
this.maxEntriesInNode = maxEntriesInNode;
}
private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) {
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE,
"The offset of the pointer to the root node must be at a well known location. "
+ "Otherwise we would not be able to find it in an already existing file.");
// 2. make sure new blocks are aligned to the block size (for faster disk IO)
diskStore.ensureAlignmentForNewBlocks(BLOCK_SIZE);
// 3. initialize an empty root node
final long blockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset);
writeNode(rootNode);
// 4. update pointer to root node
writeNodeOffsetOfRootNode(blockOffset);
// 5. insert a dummy entry with a 'maximum' key
putValue(MAX_KEY, valueEncoder.getEmptyValue());
}
}
public synchronized void putAllValues(final Map<K, V> map) {
for (final Entry<K, V> e : map.entrySet()) {
putValue(e.getKey(), e.getValue());
}
}
public synchronized V putValue(final K key, final V value) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null && cachedValue == value) {
return value;
}
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] encodedValue = valueEncoder.encode(value);
final byte[] encodedOldValue = putValue(encodedKey, encodedValue);
final V oldValue = encodedOldValue == null ? null : valueEncoder.decode(encodedOldValue);
valueCache.put(key, value);
return oldValue;
}
public synchronized V getValue(final K key) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null) {
return cachedValue;
}
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] foundValue = getValue(encodedKey);
final V result = foundValue == null ? null : valueEncoder.decode(foundValue);
valueCache.put(key, result);
return result;
}
private byte[] putValue(final byte[] key, final byte[] value) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final Stack<PersistentMapDiskNode> parents = new Stack<>();
return insert(parents, rootNodeOffset, key, value);
}
private byte[] getValue(final byte[] key) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final NodeEntry entry = findNodeEntry(rootNodeOffset, key);
return entry == null ? null : entry.getValue();
}
private byte[] insert(final Stack<PersistentMapDiskNode> parents, final long nodeOffest, final byte[] key,
final byte[] value) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final NodeEntry entry = node.getNodeEntryTo(key);
if (entry == null || entry.isDataNode()) {
final byte[] oldValue;
if (entry == null) {
oldValue = null;
} else {
// found a NodeEntry that is either equal to key, or it is at the insertion
// point
final boolean entryIsForKey = entry.equal(key);
oldValue = entryIsForKey ? entry.getValue() : null;
// Early exit, if the oldValue equals the new value.
// We do not have to replace the value, because it would not change anything
// (just cause unnecessary write operations). But we return the oldValue so that
// the caller thinks we replaced the value.
if (Objects.equals(oldValue, value)) {
return oldValue;
}
if (entryIsForKey) {
node.removeKey(key);
}
}
if (node.canAdd(key, value, maxEntriesInNode)) {
// insert in existing node
node.addKeyValue(key, value);
writeNode(node);
return oldValue;
} else {
// add new node
// 1. split current node into A and B
splitNode(parents, node);
// 2. insert the value
// start from the root, because we might have added a new root node
return putValue(key, value);
}
} else {
final long childNodeOffset = toNodeOffset(entry);
parents.add(node);
return insert(parents, childNodeOffset, key, value);
}
}
private PersistentMapDiskNode splitNode(final Stack<PersistentMapDiskNode> parents,
final PersistentMapDiskNode node) {
// System.out.println("\n\npre split node: " + node + "\n");
final long newBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode newNode = node.split(newBlockOffset);
final PersistentMapDiskNode parent = parents.isEmpty() ? null : parents.pop();
if (parent != null) {
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
if (parent.canAdd(newNodeKey, newBlockOffset, maxEntriesInNode)) {
parent.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parent);
writeNode(newNode);
writeNode(node);
return parent;
} else {
final PersistentMapDiskNode grandParentNode = splitNode(parents, parent);
final NodeEntry pointerToParentAfterSplit = grandParentNode.getNodeEntryTo(newNodeKey);
Preconditions.checkEqual(pointerToParentAfterSplit.isInnerNode(), true, "{0} is pointer to inner node",
pointerToParentAfterSplit);
final long parentNodeOffset = toNodeOffset(pointerToParentAfterSplit); // the parent we have to add the
// newNode to
final PersistentMapDiskNode parentNode = getNode(parentNodeOffset);
parentNode.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parentNode);
writeNode(newNode);
writeNode(node);
return parentNode;
}
} else {
// has no parent -> create a new parent (the new parent will also be the new
// root)
final long newRootNodeOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode rootNode = PersistentMapDiskNode.emptyRootNode(newRootNodeOffset);
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(newNodeKey, newBlockOffset);
final byte[] oldNodeKey = node.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(oldNodeKey, node.getNodeOffset());
writeNode(rootNode);
writeNode(newNode);
writeNode(node);
writeNodeOffsetOfRootNode(newRootNodeOffset);
return rootNode;
}
}
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final var entry = node.getNodeEntryTo(key);
if (entry == null) {
return null;
} else if (entry.isDataNode()) {
if (entry.equal(key)) {
return entry;
} else {
return null;
}
} else {
final long childNodeOffset = toNodeOffset(entry);
return findNodeEntry(childNodeOffset, key);
}
}
private long toNodeOffset(final NodeEntry entry) {
Preconditions.checkEqual(entry.isInnerNode(), true);
return VariableByteEncoder.decodeFirstValue(entry.getValue());
}
private PersistentMapDiskNode getNode(final long nodeOffset) {
PersistentMapDiskNode node = nodeCache.get(nodeOffset);
if (node == null) {
final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffset, BLOCK_SIZE);
node = PersistentMapDiskNode.parse(nodeOffset, diskBlock);
nodeCache.put(nodeOffset, node);
}
return node;
}
private void writeNode(final PersistentMapDiskNode node) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("writing node {}", node.toString(keyEncoder.asDecoder(), valueEncoder.asDecoder()));
}
final long nodeOffest = node.getNodeOffset();
// final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
DiskBlock diskBlock = node.getDiskBlock();
if (diskBlock == null) {
diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
}
final byte[] buffer = diskBlock.getBuffer();
final byte[] newBuffer = node.serialize();
System.arraycopy(newBuffer, 0, buffer, 0, buffer.length);
diskBlock.writeAsync();
// diskBlock.force(); // makes writing nodes slower by factor 800 (sic!)
}
public synchronized void print() {
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
final PrintStream writer = System.out;
final String children = "#" + node.getEntries().size();
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry
.toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b))));
});
}
public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
visitNodeEntriesPreOrderRecursively(rootNodeOffset, null, visitor, 0);
}
private void visitNodeEntriesPreOrderRecursively(final long nodeOffset, final PersistentMapDiskNode parentNode,
final VisitorCallback visitor, final int depth) {
final PersistentMapDiskNode node = getNode(nodeOffset);
for (final NodeEntry child : node.getEntries()) {
visitor.visit(node, parentNode, child, depth);
if (child.isInnerNode()) {
final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue());
visitNodeEntriesPreOrderRecursively(childNodeOffset, node, visitor, depth + 1);
}
}
}
enum VisitByPrefixMode {
FIND, ITERATE
}
public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) {
final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix);
final long rootNodeOffset = readNodeOffsetOfRootNode();
iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor);
}
private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) {
final PersistentMapDiskNode node = getNode(nodeOffest);
// list of children that might contain a key with the keyPrefix
final List<NodeEntry> nodesForPrefix = node.getNodesByPrefix(keyPrefix);
for (final NodeEntry entry : nodesForPrefix) {
if (entry.isDataNode()) {
final int prefixCompareResult = entry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) {
if (Arrays.equals(entry.getKey(), MAX_KEY)) {
continue;
}
final K key = keyEncoder.decode(entry.getKey());
final V value = valueEncoder.decode(entry.getValue());
visitor.visit(key, value);
// System.out.println("--> " + key + "=" + value);
} else if (prefixCompareResult > 0) {
break;
}
} else {
final long childNodeOffset = toNodeOffset(entry);
iterateNodeEntryByPrefix(childNodeOffset, keyPrefix, visitor);
}
}
}
private long readNodeOffsetOfRootNode() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
return diskBlock.getByteBuffer().getLong(0);
}
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode);
diskBlock.force();
}
@Override
public Empty decode(final byte[] bytes) {
Preconditions.checkTrue(bytes.length == 0, "");
return Empty.INSTANCE;
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
}
public static final EncoderDecoder<Long> LONG_CODER = new LongCoder();
public static final EncoderDecoder<UUID> UUID_ENCODER = new UUIDCoder();
public static final EncoderDecoder<String> STRING_CODER = new StringCoder();
public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder();
static final int BLOCK_SIZE = 4096;
static final long NODE_OFFSET_TO_ROOT_NODE = 8;
private final DiskStorage diskStore;
private int maxEntriesInNode = Integer.MAX_VALUE;
private final EncoderDecoder<K> keyEncoder;
private final EncoderDecoder<V> valueEncoder;
private final LRUCache<Long, PersistentMapDiskNode> nodeCache = new LRUCache<>(10_000);
private final LRUCache<K, V> valueCache = new LRUCache<>(1_000);
public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder,
final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path, storageBasePath);
this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder;
initIfNew();
}
@Override
public void close() {
diskStore.close();
}
public void setMaxEntriesInNode(final int maxEntriesInNode) {
this.maxEntriesInNode = maxEntriesInNode;
}
private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) {
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE,
"The offset of the pointer to the root node must be at a well known location. "
+ "Otherwise we would not be able to find it in an already existing file.");
// 2. make sure new blocks are aligned to the block size (for faster disk IO)
diskStore.ensureAlignmentForNewBlocks(BLOCK_SIZE);
// 3. initialize an empty root node
final long blockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset);
writeNode(rootNode);
// 4. update pointer to root node
writeNodeOffsetOfRootNode(blockOffset);
// 5. insert a dummy entry with a 'maximum' key
putValue(MAX_KEY, valueEncoder.getEmptyValue());
}
}
public synchronized void putAllValues(final Map<K, V> map) {
for (final Entry<K, V> e : map.entrySet()) {
putValue(e.getKey(), e.getValue());
}
}
public synchronized V putValue(final K key, final V value) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null && cachedValue == value) {
return value;
}
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] encodedValue = valueEncoder.encode(value);
final byte[] encodedOldValue = putValue(encodedKey, encodedValue);
final V oldValue = encodedOldValue == null ? null : valueEncoder.decode(encodedOldValue);
valueCache.put(key, value);
return oldValue;
}
public synchronized V getValue(final K key) {
final V cachedValue = valueCache.get(key);
if (cachedValue != null) {
return cachedValue;
}
final byte[] encodedKey = keyEncoder.encode(key);
final byte[] foundValue = getValue(encodedKey);
final V result = foundValue == null ? null : valueEncoder.decode(foundValue);
valueCache.put(key, result);
return result;
}
private byte[] putValue(final byte[] key, final byte[] value) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final Stack<PersistentMapDiskNode> parents = new Stack<>();
return insert(parents, rootNodeOffset, key, value);
}
private byte[] getValue(final byte[] key) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final NodeEntry entry = findNodeEntry(rootNodeOffset, key);
return entry == null ? null : entry.getValue();
}
private byte[] insert(final Stack<PersistentMapDiskNode> parents, final long nodeOffest, final byte[] key,
final byte[] value) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final NodeEntry entry = node.getNodeEntryTo(key);
if (entry == null || entry.isDataNode()) {
final byte[] oldValue;
if (entry == null) {
oldValue = null;
} else {
// found a NodeEntry that is either equal to key, or it is at the insertion
// point
final boolean entryIsForKey = entry.equal(key);
oldValue = entryIsForKey ? entry.getValue() : null;
// Early exit, if the oldValue equals the new value.
// We do not have to replace the value, because it would not change anything
// (just cause unnecessary write operations). But we return the oldValue so that
// the caller thinks we replaced the value.
if (Objects.equals(oldValue, value)) {
return oldValue;
}
if (entryIsForKey) {
node.removeKey(key);
}
}
if (node.canAdd(key, value, maxEntriesInNode)) {
// insert in existing node
node.addKeyValue(key, value);
writeNode(node);
return oldValue;
} else {
// add new node
// 1. split current node into A and B
splitNode(parents, node);
// 2. insert the value
// start from the root, because we might have added a new root node
return putValue(key, value);
}
} else {
final long childNodeOffset = toNodeOffset(entry);
parents.add(node);
return insert(parents, childNodeOffset, key, value);
}
}
private PersistentMapDiskNode splitNode(final Stack<PersistentMapDiskNode> parents,
final PersistentMapDiskNode node) {
// System.out.println("\n\npre split node: " + node + "\n");
final long newBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode newNode = node.split(newBlockOffset);
final PersistentMapDiskNode parent = parents.isEmpty() ? null : parents.pop();
if (parent != null) {
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
if (parent.canAdd(newNodeKey, newBlockOffset, maxEntriesInNode)) {
parent.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parent);
writeNode(newNode);
writeNode(node);
return parent;
} else {
final PersistentMapDiskNode grandParentNode = splitNode(parents, parent);
final NodeEntry pointerToParentAfterSplit = grandParentNode.getNodeEntryTo(newNodeKey);
Preconditions.checkEqual(pointerToParentAfterSplit.isInnerNode(), true, "{0} is pointer to inner node",
pointerToParentAfterSplit);
final long parentNodeOffset = toNodeOffset(pointerToParentAfterSplit); // the parent we have to add the
// newNode to
final PersistentMapDiskNode parentNode = getNode(parentNodeOffset);
parentNode.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parentNode);
writeNode(newNode);
writeNode(node);
return parentNode;
}
} else {
// has no parent -> create a new parent (the new parent will also be the new
// root)
final long newRootNodeOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode rootNode = PersistentMapDiskNode.emptyRootNode(newRootNodeOffset);
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(newNodeKey, newBlockOffset);
final byte[] oldNodeKey = node.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(oldNodeKey, node.getNodeOffset());
writeNode(rootNode);
writeNode(newNode);
writeNode(node);
writeNodeOffsetOfRootNode(newRootNodeOffset);
return rootNode;
}
}
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) {
final PersistentMapDiskNode node = getNode(nodeOffest);
final var entry = node.getNodeEntryTo(key);
if (entry == null) {
return null;
} else if (entry.isDataNode()) {
if (entry.equal(key)) {
return entry;
} else {
return null;
}
} else {
final long childNodeOffset = toNodeOffset(entry);
return findNodeEntry(childNodeOffset, key);
}
}
private long toNodeOffset(final NodeEntry entry) {
Preconditions.checkEqual(entry.isInnerNode(), true);
return VariableByteEncoder.decodeFirstValue(entry.getValue());
}
private PersistentMapDiskNode getNode(final long nodeOffset) {
PersistentMapDiskNode node = nodeCache.get(nodeOffset);
if (node == null) {
final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffset, BLOCK_SIZE);
node = PersistentMapDiskNode.parse(nodeOffset, diskBlock);
nodeCache.put(nodeOffset, node);
}
return node;
}
private void writeNode(final PersistentMapDiskNode node) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("writing node {}", node.toString(keyEncoder.asDecoder(), valueEncoder.asDecoder()));
}
final long nodeOffest = node.getNodeOffset();
// final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
DiskBlock diskBlock = node.getDiskBlock();
if (diskBlock == null) {
diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
}
final byte[] buffer = diskBlock.getBuffer();
final byte[] newBuffer = node.serialize();
System.arraycopy(newBuffer, 0, buffer, 0, buffer.length);
diskBlock.writeAsync();
// diskBlock.force(); // makes writing nodes slower by factor 800 (sic!)
}
public synchronized void print() {
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
final PrintStream writer = System.out;
final String children = "#" + node.getEntries().size();
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry
.toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b))));
});
}
public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) {
final long rootNodeOffset = readNodeOffsetOfRootNode();
visitNodeEntriesPreOrderRecursively(rootNodeOffset, null, visitor, 0);
}
private void visitNodeEntriesPreOrderRecursively(final long nodeOffset, final PersistentMapDiskNode parentNode,
final VisitorCallback visitor, final int depth) {
final PersistentMapDiskNode node = getNode(nodeOffset);
for (final NodeEntry child : node.getEntries()) {
visitor.visit(node, parentNode, child, depth);
if (child.isInnerNode()) {
final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue());
visitNodeEntriesPreOrderRecursively(childNodeOffset, node, visitor, depth + 1);
}
}
}
enum VisitByPrefixMode {
FIND, ITERATE
}
public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) {
final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix);
final long rootNodeOffset = readNodeOffsetOfRootNode();
iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor);
}
private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) {
final PersistentMapDiskNode node = getNode(nodeOffest);
// list of children that might contain a key with the keyPrefix
final List<NodeEntry> nodesForPrefix = node.getNodesByPrefix(keyPrefix);
for (final NodeEntry entry : nodesForPrefix) {
if (entry.isDataNode()) {
final int prefixCompareResult = entry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) {
if (Arrays.equals(entry.getKey(), MAX_KEY)) {
continue;
}
final K key = keyEncoder.decode(entry.getKey());
final V value = valueEncoder.decode(entry.getValue());
visitor.visit(key, value);
// System.out.println("--> " + key + "=" + value);
} else if (prefixCompareResult > 0) {
break;
}
} else {
final long childNodeOffset = toNodeOffset(entry);
iterateNodeEntryByPrefix(childNodeOffset, keyPrefix, visitor);
}
}
}
private long readNodeOffsetOfRootNode() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
return diskBlock.getByteBuffer().getLong(0);
}
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode);
diskBlock.force();
}
}

386
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java
View File

@@ -42,256 +42,256 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*/
public class PersistentMapDiskNode {
private final TreeMap<ByteArrayKey, NodeEntry> entries;
private final long nodeOffset;
private final DiskBlock diskBlock;
private final TreeMap<ByteArrayKey, NodeEntry> entries;
private final long nodeOffset;
private final DiskBlock diskBlock;
public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries,
final DiskBlock diskBlock) {
this.nodeOffset = nodeOffset;
this.diskBlock = diskBlock;
this.entries = toMap(entries);
}
public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries,
final DiskBlock diskBlock) {
this.nodeOffset = nodeOffset;
this.diskBlock = diskBlock;
this.entries = toMap(entries);
}
private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) {
final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>();
private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) {
final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>();
for (final NodeEntry nodeEntry : nodeEntries) {
result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry);
}
for (final NodeEntry nodeEntry : nodeEntries) {
result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry);
}
return result;
}
return result;
}
public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) {
return new PersistentMapDiskNode(nodeOffset, Collections.emptyList(), null);
}
public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) {
return new PersistentMapDiskNode(nodeOffset, Collections.emptyList(), null);
}
public static PersistentMapDiskNode parse(final long nodeOffset, final DiskBlock diskBlock) {
final byte[] data = diskBlock.getBuffer();
if (data.length != PersistentMap.BLOCK_SIZE) {
throw new IllegalStateException(
"block size must be " + PersistentMap.BLOCK_SIZE + " but was " + data.length);
}
final LongList longs = VariableByteEncoder.decode(data);
public static PersistentMapDiskNode parse(final long nodeOffset, final DiskBlock diskBlock) {
final byte[] data = diskBlock.getBuffer();
if (data.length != PersistentMap.BLOCK_SIZE) {
throw new IllegalStateException(
"block size must be " + PersistentMap.BLOCK_SIZE + " but was " + data.length);
}
final LongList longs = VariableByteEncoder.decode(data);
final List<NodeEntry> entries = deserialize(longs, data);
return new PersistentMapDiskNode(nodeOffset, entries, diskBlock);
}
final List<NodeEntry> entries = deserialize(longs, data);
return new PersistentMapDiskNode(nodeOffset, entries, diskBlock);
}
public static List<NodeEntry> deserialize(final LongList keyLengths, final byte[] buffer) {
final List<NodeEntry> entries = new ArrayList<>();
public static List<NodeEntry> deserialize(final LongList keyLengths, final byte[] buffer) {
final List<NodeEntry> entries = new ArrayList<>();
if (keyLengths.isEmpty() || keyLengths.get(0) == 0) {
// node is empty -> should only happen for the root node
} else {
final int numEntries = (int) keyLengths.get(0);
if (keyLengths.isEmpty() || keyLengths.get(0) == 0) {
// node is empty -> should only happen for the root node
} else {
final int numEntries = (int) keyLengths.get(0);
int offset = PersistentMap.BLOCK_SIZE;
for (int i = 0; i < numEntries; i++) {
final int keyLength = (int) keyLengths.get(i * 2 + 1);
final int valueLength = (int) keyLengths.get(i * 2 + 2);
int offset = PersistentMap.BLOCK_SIZE;
for (int i = 0; i < numEntries; i++) {
final int keyLength = (int) keyLengths.get(i * 2 + 1);
final int valueLength = (int) keyLengths.get(i * 2 + 2);
final int valueOffset = offset - valueLength;
final int keyOffset = valueOffset - keyLength;
final int typeOffset = keyOffset - 1;
final int valueOffset = offset - valueLength;
final int keyOffset = valueOffset - keyLength;
final int typeOffset = keyOffset - 1;
final byte typeByte = buffer[typeOffset];
final byte[] key = Arrays.copyOfRange(buffer, keyOffset, keyOffset + keyLength);
final byte[] value = Arrays.copyOfRange(buffer, valueOffset, valueOffset + valueLength);
final byte typeByte = buffer[typeOffset];
final byte[] key = Arrays.copyOfRange(buffer, keyOffset, keyOffset + keyLength);
final byte[] value = Arrays.copyOfRange(buffer, valueOffset, valueOffset + valueLength);
final NodeEntry entry = new NodeEntry(ValueType.fromByte(typeByte), key, value);
final NodeEntry entry = new NodeEntry(ValueType.fromByte(typeByte), key, value);
entries.add(entry);
entries.add(entry);
offset = typeOffset;
}
}
return entries;
}
offset = typeOffset;
}
}
return entries;
}
public byte[] serialize() {
public byte[] serialize() {
return serialize(entries);
}
return serialize(entries);
}
public DiskBlock getDiskBlock() {
return diskBlock;
}
public DiskBlock getDiskBlock() {
return diskBlock;
}
public long getNodeOffset() {
return nodeOffset;
}
public long getNodeOffset() {
return nodeOffset;
}
public NodeEntry getNodeEntryTo(final byte[] key) {
public NodeEntry getNodeEntryTo(final byte[] key) {
final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key));
return ceilingEntry != null ? ceilingEntry.getValue() : null;
}
final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key));
return ceilingEntry != null ? ceilingEntry.getValue() : null;
}
public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) {
final List<NodeEntry> result = new ArrayList<>();
public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) {
final List<NodeEntry> result = new ArrayList<>();
for (final NodeEntry nodeEntry : entries.values()) {
final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) {
// add all entries where keyPrefix is a prefix of the key
result.add(nodeEntry);
} else if (prefixCompareResult > 0) {
// Only add the first entry where the keyPrefix is smaller (as defined by
// compareKeyPrefix) than the key.
// These are entries that might contain key with the keyPrefix. But only the
// first of those can really have such keys.
result.add(nodeEntry);
break;
}
}
for (final NodeEntry nodeEntry : entries.values()) {
final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) {
// add all entries where keyPrefix is a prefix of the key
result.add(nodeEntry);
} else if (prefixCompareResult > 0) {
// Only add the first entry where the keyPrefix is smaller (as defined by
// compareKeyPrefix) than the key.
// These are entries that might contain key with the keyPrefix. But only the
// first of those can really have such keys.
result.add(nodeEntry);
break;
}
}
return result;
}
return result;
}
public void addKeyValue(final byte[] key, final byte[] value) {
addNode(ValueType.VALUE_INLINE, key, value);
}
public void addKeyValue(final byte[] key, final byte[] value) {
addNode(ValueType.VALUE_INLINE, key, value);
}
public void addKeyNodePointer(final byte[] key, final long nodePointer) {
final byte[] value = VariableByteEncoder.encode(nodePointer);
addNode(ValueType.NODE_POINTER, key, value);
}
public void addKeyNodePointer(final byte[] key, final long nodePointer) {
final byte[] value = VariableByteEncoder.encode(nodePointer);
addNode(ValueType.NODE_POINTER, key, value);
}
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.put(new ByteArrayKey(key), entry);
}
final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.put(new ByteArrayKey(key), entry);
}
public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) {
return canAdd(key, VariableByteEncoder.encode(nodeOffset), maxEntriesInNode);
}
public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) {
return canAdd(key, VariableByteEncoder.encode(nodeOffset), maxEntriesInNode);
}
public boolean canAdd(final byte[] key, final byte[] value, final int maxEntriesInNode) {
public boolean canAdd(final byte[] key, final byte[] value, final int maxEntriesInNode) {
if (entries.size() > maxEntriesInNode) {
return false;
} else {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries.values());
tmp.add(entry);
if (entries.size() > maxEntriesInNode) {
return false;
} else {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries.values());
tmp.add(entry);
// the +1 is for the null-byte terminator of the prefix
return neededBytesTotal(tmp) + 1 <= PersistentMap.BLOCK_SIZE;
}
}
// the +1 is for the null-byte terminator of the prefix
return neededBytesTotal(tmp) + 1 <= PersistentMap.BLOCK_SIZE;
}
}
public void removeKey(final byte[] key) {
entries.remove(new ByteArrayKey(key));
}
public void removeKey(final byte[] key) {
entries.remove(new ByteArrayKey(key));
}
public List<NodeEntry> getEntries() {
return new ArrayList<>(entries.values());
}
public List<NodeEntry> getEntries() {
return new ArrayList<>(entries.values());
}
public void clear() {
entries.clear();
}
public void clear() {
entries.clear();
}
@Override
public String toString() {
return "@" + nodeOffset + ": "
+ String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList()));
}
public <K,V> String toString(Function<byte[], K> keyDecoder, Function<byte[], V> valueDecoder) {
StringBuilder result = new StringBuilder();
result.append("@");
result.append(nodeOffset);
result.append(": ");
for (NodeEntry e : entries.values()) {
String s = e.toString(keyDecoder, valueDecoder);
result.append("\n");
result.append(s);
}
return result.toString();
}
@Override
public String toString() {
return "@" + nodeOffset + ": "
+ String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList()));
}
public NodeEntry getTopNodeEntry() {
return entries.lastEntry().getValue();
}
public <K, V> String toString(Function<byte[], K> keyDecoder, Function<byte[], V> valueDecoder) {
StringBuilder result = new StringBuilder();
result.append("@");
result.append(nodeOffset);
result.append(": ");
for (NodeEntry e : entries.values()) {
String s = e.toString(keyDecoder, valueDecoder);
result.append("\n");
result.append(s);
}
public PersistentMapDiskNode split(final long newBlockOffset) {
return result.toString();
}
final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values());
public NodeEntry getTopNodeEntry() {
return entries.lastEntry().getValue();
}
final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2));
final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
public PersistentMapDiskNode split(final long newBlockOffset) {
entries.clear();
entries.putAll(toMap(rightEntries));
final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values());
return new PersistentMapDiskNode(newBlockOffset, leftEntries, null);
}
final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2));
final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
public static int neededBytesTotal(final List<NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
entries.clear();
entries.putAll(toMap(rightEntries));
final int usedBytes = serializePrefix(entries, buffer);
return new PersistentMapDiskNode(newBlockOffset, leftEntries, null);
}
return usedBytes + NodeEntry.neededBytes(entries);
}
public static int neededBytesTotal(final List<NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entriesAsCollection, buffer);
final int usedBytes = serializePrefix(entries, buffer);
// the +1 is for the null-byte terminator of the prefix
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE,
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
return usedBytes + NodeEntry.neededBytes(entries);
}
serializeIntoFromTail(entriesAsCollection, buffer);
return buffer;
}
private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) {
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entriesAsCollection, buffer);
private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) {
final LongList longs = serializeKeyLengths(entries);
// the +1 is for the null-byte terminator of the prefix
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE,
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0);
return usedBytes;
}
serializeIntoFromTail(entriesAsCollection, buffer);
return buffer;
}
private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) {
final var keyLengths = new LongList();
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
keyLengths.add(nodeEntry.getKey().length);
keyLengths.add(nodeEntry.getValue().length);
}
private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) {
final LongList longs = serializeKeyLengths(entries);
return keyLengths;
}
final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0);
return usedBytes;
}
private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) {
private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) {
final var keyLengths = new LongList();
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
keyLengths.add(nodeEntry.getKey().length);
keyLengths.add(nodeEntry.getValue().length);
}
int offset = buffer.length;
return keyLengths;
}
for (final var entry : entries) {
final byte[] valueBytes = entry.getValue();
final byte[] keyBytes = entry.getKey();
private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) {
final int offsetValue = offset - valueBytes.length;
final int offsetKey = offsetValue - keyBytes.length;
final int offsetType = offsetKey - 1;
int offset = buffer.length;
System.arraycopy(valueBytes, 0, buffer, offsetValue, valueBytes.length);
System.arraycopy(keyBytes, 0, buffer, offsetKey, keyBytes.length);
buffer[offsetType] = entry.getType().asByte();
for (final var entry : entries) {
final byte[] valueBytes = entry.getValue();
final byte[] keyBytes = entry.getKey();
offset = offsetType;
}
}
final int offsetValue = offset - valueBytes.length;
final int offsetKey = offsetValue - keyBytes.length;
final int offsetType = offsetKey - 1;
System.arraycopy(valueBytes, 0, buffer, offsetValue, valueBytes.length);
System.arraycopy(keyBytes, 0, buffer, offsetKey, keyBytes.length);
buffer[offsetType] = entry.getType().asByte();
offset = offsetType;
}
}
}

2
block-storage/src/main/java/org/lucares/pdb/map/Visitor.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.map;
public interface Visitor<K, V> {
void visit(K key, V value);
void visit(K key, V value);
}

168
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -25,110 +25,110 @@ import org.testng.annotations.Test;
@Test
public class BSFileTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testBlockStorage() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final int numLongs = 1000;
long blockOffset = -1;
public void testBlockStorage() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final int numLongs = 1000;
long blockOffset = -1;
long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
long start = System.nanoTime();
try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
blockOffset = bsFile.getRootBlockOffset();
try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
for (long i = 0; i < numLongs / 2; i++) {
bsFile.append(i);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE)) {
blockOffset = bsFile.getRootBlockOffset();
for (long i = numLongs / 2; i < numLongs; i++) {
bsFile.append(i);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
for (long i = 0; i < numLongs / 2; i++) {
bsFile.append(i);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE)) {
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
for (long i = numLongs / 2; i < numLongs; i++) {
bsFile.append(i);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
public void testBlockStorageMultithreading() throws Exception {
final ExecutorService pool = Executors.newCachedThreadPool();
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
final Path file = dataDirectory.resolve("data.int.db");
public void testBlockStorageMultithreading() throws Exception {
final ExecutorService pool = Executors.newCachedThreadPool();
final int threads = 50;
final int values = 10000;
final Map<Long, LongList> expected = new HashMap<>();
final List<Future<Void>> futures = new ArrayList<>();
final long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final Path file = dataDirectory.resolve("data.int.db");
for (int i = 0; i < threads; i++) {
final Future<Void> future = pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
final LongList listOfValues = new LongList();
final int threads = 50;
final int values = 10000;
final Map<Long, LongList> expected = new HashMap<>();
final List<Future<Void>> futures = new ArrayList<>();
final long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
for (int i = 0; i < threads; i++) {
final Future<Void> future = pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
final LongList listOfValues = new LongList();
for (int j = 0; j < values; j++) {
try (BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
// will produce 1,2 and 3 byte sequences when encoded
final long value = random.nextLong(32768);
listOfValues.add(value);
bsFile.append(value);
}
expected.put(bsFile.getRootBlockOffset(), listOfValues);
}
for (int j = 0; j < values; j++) {
return null;
});
futures.add(future);
}
// will produce 1,2 and 3 byte sequences when encoded
final long value = random.nextLong(32768);
listOfValues.add(value);
bsFile.append(value);
}
expected.put(bsFile.getRootBlockOffset(), listOfValues);
}
for (final Future<Void> future : futures) {
future.get();
}
return null;
});
futures.add(future);
}
pool.shutdown();
pool.awaitTermination(5, TimeUnit.MINUTES);
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
for (final Future<Void> future : futures) {
future.get();
}
// verification
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
for (final Entry<Long, LongList> entry : expected.entrySet()) {
final long rootBlockNumber = entry.getKey();
final LongList expectedValues = entry.getValue();
pool.shutdown();
pool.awaitTermination(5, TimeUnit.MINUTES);
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds, NullCustomizer.INSTANCE)) {
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = expectedValues;
Assert.assertEquals(actualLongs, expectedLongs, "for rootBlockNumber=" + rootBlockNumber);
}
}
}
}
// verification
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
for (final Entry<Long, LongList> entry : expected.entrySet()) {
final long rootBlockNumber = entry.getKey();
final LongList expectedValues = entry.getValue();
try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds, NullCustomizer.INSTANCE)) {
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = expectedValues;
Assert.assertEquals(actualLongs, expectedLongs, "for rootBlockNumber=" + rootBlockNumber);
}
}
}
}
}

100
block-storage/src/test/java/org/lucares/pdb/blockstorage/TimeSeriesFileTest.java
View File

@@ -15,70 +15,70 @@ import org.testng.annotations.BeforeMethod;
public class TimeSeriesFileTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testBlockStorageTimeValue() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final Random random = ThreadLocalRandom.current();
final int numTimeValuePairs = 1000;
long blockNumber = -1;
final LongList expectedLongs = new LongList();
public void testBlockStorageTimeValue() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final Random random = ThreadLocalRandom.current();
final int numTimeValuePairs = 1000;
long blockNumber = -1;
final LongList expectedLongs = new LongList();
long start = System.nanoTime();
long lastEpochMilli = 0;
//
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
long start = System.nanoTime();
long lastEpochMilli = 0;
//
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) {
try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockOffset();
blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) {
for (long i = 0; i < numTimeValuePairs / 2; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(1000);
final long value = random.nextInt(10000);
final long epochMilli = lastEpochMilli + random.nextInt(1000);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
try (final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds)) {
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
try (final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds)) {
for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(100);
final long value = random.nextInt(10000);
for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(100);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList();
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList();
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}

450
block-storage/src/test/java/org/lucares/pdb/diskstorage/DiskStorageTest.java
View File

@@ -18,289 +18,289 @@ import org.testng.annotations.Test;
@Test
public class DiskStorageTest {
private static final int BLOCK_SIZE = 512;
private static final int BLOCK_SIZE = 512;
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
/**
* File systems work with 4096 byte blocks, but we want to work with 512 bytes
* per block. Does flushing a 512 byte block flush the full 4096 byte block?
*
* @throws Exception
*/
@Test(enabled = false)
public void testFlushingASectorOrABlock() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
Files.deleteIfExists(databaseFile);
/**
* File systems work with 4096 byte blocks, but we want to work with 512 bytes
* per block. Does flushing a 512 byte block flush the full 4096 byte block?
*
* @throws Exception
*/
@Test(enabled = false)
public void testFlushingASectorOrABlock() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
Files.deleteIfExists(databaseFile);
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
allocateBlocks(ds, numBlocks, BLOCK_SIZE);
final List<DiskBlock> blocks = new ArrayList<>();
allocateBlocks(ds, numBlocks, BLOCK_SIZE);
final List<DiskBlock> blocks = new ArrayList<>();
// fill the first 16 512-byte blocks
// that is more than on 4096 byte block
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) i);
diskBlock.writeAsync();
blocks.add(diskBlock);
}
// fill the first 16 512-byte blocks
// that is more than on 4096 byte block
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) i);
diskBlock.writeAsync();
blocks.add(diskBlock);
}
// now force (aka flush) a block in the middle of the first 4096 byte block
blocks.get(3).writeAsync();
blocks.get(3).force();
// now force (aka flush) a block in the middle of the first 4096 byte block
blocks.get(3).writeAsync();
blocks.get(3).force();
System.exit(0);
System.exit(0);
// read all blocks again an check what they contain
// read all blocks again an check what they contain
// 1. we do this with the existing file channel
// this one should see every change, because we wrote them to the file channel
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
}
// 1. we do this with the existing file channel
// this one should see every change, because we wrote them to the file channel
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
}
// 2. we read the file from another file channel
// this one might not see changes made to the first file channel
//
// But it does see the changes. Most likely, because both channels
// use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile, dataDirectory)) {
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
}
}
}
}
// 2. we read the file from another file channel
// this one might not see changes made to the first file channel
//
// But it does see the changes. Most likely, because both channels
// use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile, dataDirectory)) {
for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i);
blocks.add(diskBlock);
}
}
}
}
@Test(enabled = true)
public void testDiskStorage() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true)
public void testDiskStorage() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
final ExecutorService pool = Executors.newCachedThreadPool();
final ExecutorService pool = Executors.newCachedThreadPool();
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10;
final long[] blockOffsets = allocateBlocks(ds, numBlocks, BLOCK_SIZE);
final long[] blockOffsets = allocateBlocks(ds, numBlocks, BLOCK_SIZE);
for (final long blockOffset : blockOffsets) {
for (final long blockOffset : blockOffsets) {
final long block = blockOffset;
pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
try {
// now read/write random blocks
for (int j = 0; j < 10; j++) {
final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE);
final long block = blockOffset;
pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
try {
// now read/write random blocks
for (int j = 0; j < 10; j++) {
final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) random.nextInt(127));
assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) random.nextInt(127));
if (random.nextBoolean()) {
diskBlock.writeAsync();
} else {
diskBlock.writeAsync();
diskBlock.force();
}
}
if (random.nextBoolean()) {
diskBlock.writeAsync();
} else {
diskBlock.writeAsync();
diskBlock.force();
}
}
} catch (final Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
}
} catch (final Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
}
pool.shutdown();
pool.awaitTermination(1, TimeUnit.MINUTES);
}
}
pool.shutdown();
pool.awaitTermination(1, TimeUnit.MINUTES);
}
}
@Test(enabled = true, expectedExceptions = IllegalArgumentException.class)
public void testAllocationSmallerThanMinimalBlockSize() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true, expectedExceptions = IllegalArgumentException.class)
public void testAllocationSmallerThanMinimalBlockSize() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 31; // minimal block size is 32
ds.allocateBlock(blockSize);
}
}
final int blockSize = 31; // minimal block size is 32
ds.allocateBlock(blockSize);
}
}
@Test(enabled = true)
public void testAllocateAndFreeSingleBlockInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true)
public void testAllocateAndFreeSingleBlockInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
final long block_8_39 = ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
final int blockSize = 32;
final long block_8_39 = ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(block_8_39, 8);
Assert.assertEquals(block_40_71, 40);
Assert.assertEquals(block_72_103, 72);
Assert.assertEquals(block_8_39, 8);
Assert.assertEquals(block_40_71, 40);
Assert.assertEquals(block_72_103, 72);
ds.free(block_40_71, blockSize);
ds.free(block_40_71, blockSize);
// should reuse the block we just freed
final long actual_block_40_71 = ds.allocateBlock(blockSize);
// should reuse the block we just freed
final long actual_block_40_71 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_40_71, 40);
}
}
Assert.assertEquals(actual_block_40_71, 40);
}
}
@Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize);
final int blockSize = 32;
ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize);
ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize);
ds.free(block_40_71, blockSize); // the block with the smaller index is freed last, this increases line
// coverage, because there is a branch for prepending the root node
ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize);
ds.free(block_40_71, blockSize); // the block with the smaller index is freed last, this increases line
// coverage, because there is a branch for prepending the root node
// should reuse the first block we just freed
// this removes the root node of the free list
final long actual_block_40_71 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_40_71, 40);
// should reuse the first block we just freed
// this removes the root node of the free list
final long actual_block_40_71 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_40_71, 40);
// should reuse the second block we just freed
final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, 72);
// should reuse the second block we just freed
final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, 72);
// should reuse the third block we just freed
// this removes the last node of the free list
final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, 104);
// should reuse the third block we just freed
// this removes the last node of the free list
final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, 104);
final long block_168_199 = ds.allocateBlock(blockSize);
Assert.assertEquals(block_168_199, 168);
}
}
final long block_168_199 = ds.allocateBlock(blockSize);
Assert.assertEquals(block_168_199, 168);
}
}
@Test(enabled = true)
public void testAllocateAndFreeInsertFreeNodeInTheMiddleOfTheFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true)
public void testAllocateAndFreeInsertFreeNodeInTheMiddleOfTheFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32;
ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize);
final long block_136_167 = ds.allocateBlock(blockSize);
final int blockSize = 32;
ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize);
final long block_136_167 = ds.allocateBlock(blockSize);
// free the last block first, to increase code coverage
ds.free(block_136_167, blockSize);
ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize);
// free the last block first, to increase code coverage
ds.free(block_136_167, blockSize);
ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize);
// the first free block is re-used
final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, block_72_103);
// the first free block is re-used
final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, block_72_103);
final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, block_104_135);
final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, block_104_135);
final long actual_block_136_167 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_136_167, block_136_167);
}
}
final long actual_block_136_167 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_136_167, block_136_167);
}
}
@Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksWithDifferentSizes() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
@Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksWithDifferentSizes() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSizeSmall = 32;
final int blockSizeBig = 64;
ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall);
final long big_block_72_103 = ds.allocateBlock(blockSizeBig);
final long small_block_136_167 = ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall);
final int blockSizeSmall = 32;
final int blockSizeBig = 64;
ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall);
final long big_block_72_103 = ds.allocateBlock(blockSizeBig);
final long small_block_136_167 = ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall);
ds.free(big_block_72_103, blockSizeBig);
ds.free(small_block_136_167, blockSizeSmall);
ds.free(big_block_72_103, blockSizeBig);
ds.free(small_block_136_167, blockSizeSmall);
final long actual_small_block_136_167 = ds.allocateBlock(blockSizeSmall);
Assert.assertEquals(actual_small_block_136_167, small_block_136_167);
}
}
final long actual_small_block_136_167 = ds.allocateBlock(blockSizeSmall);
Assert.assertEquals(actual_small_block_136_167, small_block_136_167);
}
}
private void assertAllValuesAreEqual(final DiskBlock diskBlock, final byte expectedVal) {
final byte[] buffer = diskBlock.getBuffer();
for (int i = 0; i < buffer.length; i++) {
if (expectedVal != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i);
break;
}
}
}
private void assertAllValuesAreEqual(final DiskBlock diskBlock, final byte expectedVal) {
final byte[] buffer = diskBlock.getBuffer();
for (int i = 0; i < buffer.length; i++) {
if (expectedVal != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i);
break;
}
}
}
private void assertAllValuesAreEqual(final DiskBlock diskBlock) {
private void assertAllValuesAreEqual(final DiskBlock diskBlock) {
final byte[] buffer = diskBlock.getBuffer();
final byte expected = buffer[0];
for (int i = 0; i < buffer.length; i++) {
if (expected != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i);
break;
}
}
final byte[] buffer = diskBlock.getBuffer();
final byte expected = buffer[0];
for (int i = 0; i < buffer.length; i++) {
if (expected != buffer[i]) {
System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i);
break;
}
}
}
}
private void fill(final DiskBlock diskBlock, final byte val) {
final byte[] buffer = diskBlock.getBuffer();
private void fill(final DiskBlock diskBlock, final byte val) {
final byte[] buffer = diskBlock.getBuffer();
for (int i = 0; i < buffer.length; i++) {
buffer[i] = val;
}
}
for (int i = 0; i < buffer.length; i++) {
buffer[i] = val;
}
}
private long[] allocateBlocks(final DiskStorage ds, final int numNewBlocks, final int blockSize)
throws IOException {
private long[] allocateBlocks(final DiskStorage ds, final int numNewBlocks, final int blockSize)
throws IOException {
final long[] result = new long[numNewBlocks];
for (int i = 0; i < numNewBlocks; i++) {
final long blockOffset = ds.allocateBlock(blockSize);
result[i] = blockOffset;
}
return result;
}
final long[] result = new long[numNewBlocks];
for (int i = 0; i < numNewBlocks; i++) {
final long blockOffset = ds.allocateBlock(blockSize);
result[i] = blockOffset;
}
return result;
}
}

132
block-storage/src/test/java/org/lucares/pdb/map/CsvTestDataCreator.java
View File

@@ -15,79 +15,79 @@ import java.util.concurrent.ThreadLocalRandom;
public class CsvTestDataCreator {
private static final List<String> PODS = Arrays.asList("vapbrewe01", "vapfinra01", "vapondem01", "vapondem02",
"vapondem03", "vapondem04", "vapnyse01", "vapnorto01", "vapfackb01", "vaprjrey01", "vadtrans01",
"vadaxcel09", "vadaxcel66");
private static final List<String> HOSTS = new ArrayList<>();
private static final List<String> CLASSES = Arrays.asList("AuditLog", "Brava", "Collection", "Folder", "Field",
"Tagging", "Arrangment", "Review", "Production", "ProductionExport", "View", "Jobs", "Navigation",
"RecentNavigation", "Entity", "Search", "Tasks", "PcWorkflow", "Batch", "Matter");
private static final List<String> ENDPOINTS = Arrays.asList("create", "remove", "update", "delete", "createBulk",
"removeBulk", "deleteBulk", "list", "index", "listing", "all");
private static final List<String> METHODS = new ArrayList<>();
private static final List<String> PROJECTS = new ArrayList<>();
private static final List<String> SOURCE = Arrays.asList("web", "service", "metrics");
private static final List<String> BUILDS = new ArrayList<>();
private static final List<String> PODS = Arrays.asList("vapbrewe01", "vapfinra01", "vapondem01", "vapondem02",
"vapondem03", "vapondem04", "vapnyse01", "vapnorto01", "vapfackb01", "vaprjrey01", "vadtrans01",
"vadaxcel09", "vadaxcel66");
private static final List<String> HOSTS = new ArrayList<>();
private static final List<String> CLASSES = Arrays.asList("AuditLog", "Brava", "Collection", "Folder", "Field",
"Tagging", "Arrangment", "Review", "Production", "ProductionExport", "View", "Jobs", "Navigation",
"RecentNavigation", "Entity", "Search", "Tasks", "PcWorkflow", "Batch", "Matter");
private static final List<String> ENDPOINTS = Arrays.asList("create", "remove", "update", "delete", "createBulk",
"removeBulk", "deleteBulk", "list", "index", "listing", "all");
private static final List<String> METHODS = new ArrayList<>();
private static final List<String> PROJECTS = new ArrayList<>();
private static final List<String> SOURCE = Arrays.asList("web", "service", "metrics");
private static final List<String> BUILDS = new ArrayList<>();
static {
for (int i = 0; i < 500; i++) {
BUILDS.add("AXC_5.15_" + i);
}
static {
for (int i = 0; i < 500; i++) {
BUILDS.add("AXC_5.15_" + i);
}
for (int i = 0; i < 500; i++) {
HOSTS.add(UUID.randomUUID().toString().substring(1, 16));
PROJECTS.add(UUID.randomUUID().toString().substring(1, 16) + "_Review");
}
for (int i = 0; i < 500; i++) {
HOSTS.add(UUID.randomUUID().toString().substring(1, 16));
PROJECTS.add(UUID.randomUUID().toString().substring(1, 16) + "_Review");
}
for (final String clazz : CLASSES) {
for (final String endpoint : ENDPOINTS) {
METHODS.add(clazz + "Service." + endpoint);
METHODS.add(clazz + "Controller." + endpoint);
}
}
}
for (final String clazz : CLASSES) {
for (final String endpoint : ENDPOINTS) {
METHODS.add(clazz + "Service." + endpoint);
METHODS.add(clazz + "Controller." + endpoint);
}
}
}
public static void main(final String[] args) throws IOException {
final Path testdataFile = Files.createTempFile("testData", ".csv");
public static void main(final String[] args) throws IOException {
final Path testdataFile = Files.createTempFile("testData", ".csv");
final ThreadLocalRandom r = ThreadLocalRandom.current();
int lines = 0;
final ThreadLocalRandom r = ThreadLocalRandom.current();
int lines = 0;
try (FileWriter writer = new FileWriter(testdataFile.toFile())) {
writer.append("@timestamp,duration,pod,host,method,project,source,build\n");
try (FileWriter writer = new FileWriter(testdataFile.toFile())) {
writer.append("@timestamp,duration,pod,host,method,project,source,build\n");
for (lines = 0; lines < 1_000_000; lines++) {
final String timestamp = Instant.ofEpochMilli(r.nextLong(1234567890L, 12345678901L))
.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
final String duration = String.valueOf(r.nextInt(10000));
final String pod = PODS.get(r.nextInt(PODS.size()));
final String host = HOSTS.get(r.nextInt(HOSTS.size()));
final String method = METHODS.get(r.nextInt(METHODS.size()));
final String project = PROJECTS.get(r.nextInt(PROJECTS.size()));
final String source = SOURCE.get(r.nextInt(SOURCE.size()));
final String build = BUILDS.get(r.nextInt(BUILDS.size()));
for (lines = 0; lines < 1_000_000; lines++) {
final String timestamp = Instant.ofEpochMilli(r.nextLong(1234567890L, 12345678901L))
.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
final String duration = String.valueOf(r.nextInt(10000));
final String pod = PODS.get(r.nextInt(PODS.size()));
final String host = HOSTS.get(r.nextInt(HOSTS.size()));
final String method = METHODS.get(r.nextInt(METHODS.size()));
final String project = PROJECTS.get(r.nextInt(PROJECTS.size()));
final String source = SOURCE.get(r.nextInt(SOURCE.size()));
final String build = BUILDS.get(r.nextInt(BUILDS.size()));
writer.append(timestamp);
writer.append(",");
writer.append(duration);
writer.append(",");
writer.append(pod);
writer.append(",");
writer.append(host);
writer.append(",");
writer.append(method);
writer.append(",");
writer.append(project);
writer.append(",");
writer.append(source);
writer.append(",");
writer.append(build);
writer.append("\n");
writer.append(timestamp);
writer.append(",");
writer.append(duration);
writer.append(",");
writer.append(pod);
writer.append(",");
writer.append(host);
writer.append(",");
writer.append(method);
writer.append(",");
writer.append(project);
writer.append(",");
writer.append(source);
writer.append(",");
writer.append(build);
writer.append("\n");
if (lines % 1000 == 0) {
System.out.println("lines: " + lines);
}
}
}
}
if (lines % 1000 == 0) {
System.out.println("lines: " + lines);
}
}
}
}
}

36
block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java
View File

@@ -11,27 +11,27 @@ import org.testng.annotations.Test;
@Test
public class NodeEntryTest {
@DataProvider
public Object[][] providerPrefixCompare() {
final List<Object[]> result = new ArrayList<>();
@DataProvider
public Object[][] providerPrefixCompare() {
final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { "ab", "abc", -1 });
result.add(new Object[] { "abb", "abc", -1 });
result.add(new Object[] { "abc", "abc", 0 });
result.add(new Object[] { "abcd", "abc", 0 });
result.add(new Object[] { "abd", "abc", 1 });
result.add(new Object[] { "abz", "abc", 23 });
result.add(new Object[] { "ab", "abc", -1 });
result.add(new Object[] { "abb", "abc", -1 });
result.add(new Object[] { "abc", "abc", 0 });
result.add(new Object[] { "abcd", "abc", 0 });
result.add(new Object[] { "abd", "abc", 1 });
result.add(new Object[] { "abz", "abc", 23 });
return result.toArray(Object[][]::new);
}
return result.toArray(Object[][]::new);
}
@Test(dataProvider = "providerPrefixCompare")
public void testPrefixCompare(final String key, final String prefix, final int expected) {
@Test(dataProvider = "providerPrefixCompare")
public void testPrefixCompare(final String key, final String prefix, final int expected) {
final NodeEntry nodeEntry = new NodeEntry(ValueType.NODE_POINTER, key.getBytes(StandardCharsets.UTF_8),
new byte[0]);
final NodeEntry nodeEntry = new NodeEntry(ValueType.NODE_POINTER, key.getBytes(StandardCharsets.UTF_8),
new byte[0]);
final int actual = nodeEntry.compareKeyPrefix(prefix.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(actual, expected, key + " ? " + prefix);
}
final int actual = nodeEntry.compareKeyPrefix(prefix.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(actual, expected, key + " ? " + prefix);
}
}

36
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapDiskNodeTest.java
View File

@@ -14,29 +14,29 @@ import org.testng.annotations.Test;
@Test
public class PersistentMapDiskNodeTest {
public void serializeDeserialize() throws Exception {
public void serializeDeserialize() throws Exception {
final List<NodeEntry> entries = new ArrayList<>();
entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1"));
entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--"));
entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---"));
entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----"));
final List<NodeEntry> entries = new ArrayList<>();
entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1"));
entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--"));
entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---"));
entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----"));
final long nodeOffset = ThreadLocalRandom.current().nextInt();
final PersistentMapDiskNode node = new PersistentMapDiskNode(nodeOffset, entries, null);
final long nodeOffset = ThreadLocalRandom.current().nextInt();
final PersistentMapDiskNode node = new PersistentMapDiskNode(nodeOffset, entries, null);
final byte[] buffer = node.serialize();
final byte[] buffer = node.serialize();
final ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
final PersistentMapDiskNode actualNode = PersistentMapDiskNode.parse(nodeOffset,
new DiskBlock(nodeOffset, byteBuffer));
final ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
final PersistentMapDiskNode actualNode = PersistentMapDiskNode.parse(nodeOffset,
new DiskBlock(nodeOffset, byteBuffer));
Assert.assertEquals(actualNode.getEntries(), entries);
}
Assert.assertEquals(actualNode.getEntries(), entries);
}
private static NodeEntry newNode(final ValueType type, final String key, final String value) {
return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8),
value.getBytes(StandardCharsets.UTF_8));
}
private static NodeEntry newNode(final ValueType type, final String key, final String value) {
return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8),
value.getBytes(StandardCharsets.UTF_8));
}
}

711
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -24,368 +24,369 @@ import org.testng.annotations.Test;
@Test
public class PersistentMapTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testSingleValue() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final String value = "value1";
final String key = "key1";
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testSingleValue() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final String value = "value1";
final String key = "key1";
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
Assert.assertNull(map.putValue(key, value));
Assert.assertEquals(map.getValue(key), value);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
Assert.assertEquals(map.getValue(key), value);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
Assert.assertEquals(map.getValue(key), value);
}
}
Assert.assertEquals(map.getValue(key), value);
}
}
@Test(invocationCount = 1)
public void testManyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
@Test(invocationCount = 1)
public void testManyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Random rnd = new Random(1);
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
map.setMaxEntriesInNode(2);
for (int i = 0; i < 100; i++) {
// System.out.println("\n\ninserting: " + i);
final UUID nextUUID = new UUID(rnd.nextLong(), rnd.nextLong());
final String key = nextUUID.toString() + "__" + i;
final String value = "long value to waste some bytes " + i + "__"
+ UUID.randomUUID().toString().repeat(1);
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> maxDepth.set(Math.max(depth, maxDepth.get())));
Assert.assertTrue(maxDepth.get() >= 4,
"The tree's depth. This test must have at least depth 4, "
+ "so that we can be sure that splitting parent nodes works recursively, but was "
+ maxDepth.get());
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testManySmallValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1000; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print();
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testManyEmptyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Empty>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
for (int i = 0; i < 1500; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Empty value = Empty.INSTANCE;
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print();
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
map.print();
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testEasyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) {
final Integer keyNumber = numbers.poll();
// System.out.println("\n\ninserting: " + keyNumber);
final String key = "" + keyNumber;
final String value = "value";
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test
public void testFindAllByPrefix() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final Map<String, String> expectedBar = new HashMap<>();
for (int i = 0; i < 100; i++) {
// the value is a little bit longer to make sure that the values don't fit into
// a single leaf node
expectedBar.put("bar:" + i, "bar:" + i + "__##################################");
}
final Map<String, String> input = new HashMap<>();
input.putAll(expectedBar);
for (int i = 0; i < 500; i++) {
input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
map.putAllValues(input);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) {
{
final LinkedHashMap<String, String> actualBar = new LinkedHashMap<>();
final Visitor<String, String> visitor = (key, value) -> actualBar.put(key, value);
map.visitValues("bar:", visitor);
Assert.assertEquals(actualBar, expectedBar);
}
}
}
@Test(invocationCount = 1)
public void testLotsOfValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1_000; i++) {
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
if (insertedValues.containsKey(key)) {
continue;
}
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
final AtomicInteger counter = new AtomicInteger();
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0);
maxDepth.set(Math.max(maxDepth.get(), depth));
});
final long start = System.nanoTime();
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
System.out.println("nodes=" + counter.get() + ", depth=" + maxDepth.get() + ": "
+ (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}
final Random rnd = new Random(1);
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
map.setMaxEntriesInNode(2);
for (int i = 0; i < 100; i++) {
// System.out.println("\n\ninserting: " + i);
final UUID nextUUID = new UUID(rnd.nextLong(), rnd.nextLong());
final String key = nextUUID.toString() + "__" + i;
final String value = "long value to waste some bytes " + i + "__"
+ UUID.randomUUID().toString().repeat(1);
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> maxDepth.set(Math.max(depth, maxDepth.get())));
Assert.assertTrue(maxDepth.get() >= 4,
"The tree's depth. This test must have at least depth 4, "
+ "so that we can be sure that splitting parent nodes works recursively, but was "
+ maxDepth.get());
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testManySmallValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1000; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print();
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testManyEmptyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Empty>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
for (int i = 0; i < 1500; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Empty value = Empty.INSTANCE;
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print();
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
map.print();
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test(invocationCount = 1)
public void testEasyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) {
final Integer keyNumber = numbers.poll();
// System.out.println("\n\ninserting: " + keyNumber);
final String key = "" + keyNumber;
final String value = "value";
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
}
}
@Test
public void testFindAllByPrefix() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final Map<String, String> expectedBar = new HashMap<>();
for (int i = 0; i < 100; i++) {
// the value is a little bit longer to make sure that the values don't fit into
// a single leaf node
expectedBar.put("bar:" + i, "bar:" + i + "__##################################");
}
final Map<String, String> input = new HashMap<>();
input.putAll(expectedBar);
for (int i = 0; i < 500; i++) {
input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
map.putAllValues(input);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
{
final LinkedHashMap<String, String> actualBar = new LinkedHashMap<>();
final Visitor<String, String> visitor = (key, value) -> actualBar.put(key, value);
map.visitValues("bar:", visitor);
Assert.assertEquals(actualBar, expectedBar);
}
}
}
@Test(invocationCount = 1)
public void testLotsOfValues() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1_000; i++) {
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
if (insertedValues.containsKey(key)) {
continue;
}
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
final boolean failEarly = false;
if (failEarly) {
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration");
}
}
}
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
final AtomicInteger counter = new AtomicInteger();
final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0);
maxDepth.set(Math.max(maxDepth.get(), depth));
});
final long start = System.nanoTime();
for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations");
}
System.out.println("nodes=" + counter.get() + ", depth=" + maxDepth.get() + ": "
+ (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}
}

348
byte-utils/src/main/java/org/lucares/utils/byteencoder/VariableByteEncoder.java
View File

@@ -25,218 +25,218 @@ import org.lucares.collections.LongList;
*/
public class VariableByteEncoder {
public static final long MIN_VALUE = Long.MIN_VALUE / 2 + 1;
public static final long MAX_VALUE = Long.MAX_VALUE / 2;
public static final long MIN_VALUE = Long.MIN_VALUE / 2 + 1;
public static final long MAX_VALUE = Long.MAX_VALUE / 2;
private static final int MAX_BYTES_PER_VALUE = 10;
private static final int MAX_BYTES_PER_VALUE = 10;
private static final int CONTINUATION_BYTE_FLAG = 1 << 7; // 10000000
private static final int CONTINUATION_BYTE_FLAG = 1 << 7; // 10000000
private static final long DATA_BITS = (1 << 7) - 1; // 01111111
private static final long DATA_BITS = (1 << 7) - 1; // 01111111
private static final ThreadLocal<byte[]> SINGLE_VALUE_BUFFER = ThreadLocal
.withInitial(() -> new byte[MAX_BYTES_PER_VALUE]);
private static final ThreadLocal<byte[]> SINGLE_VALUE_BUFFER = ThreadLocal
.withInitial(() -> new byte[MAX_BYTES_PER_VALUE]);
/**
* Encodes time and value into the given buffer.
* <p>
* If the encoded values do not fit into the buffer, then 0 is returned. The
* caller will have to provide a new buffer with more space.
*
* @param value1 first value, (between -(2^62)+1 and 2^62)
* @param value2 second value, (between -(2^62)+1 and 2^62)
* @param buffer
* @param offsetInBuffer
* @return number of bytes appended to the provided buffer
*/
public static int encodeInto(final long value1, final long value2, final byte[] buffer, final int offsetInBuffer) {
/**
* Encodes time and value into the given buffer.
* <p>
* If the encoded values do not fit into the buffer, then 0 is returned. The
* caller will have to provide a new buffer with more space.
*
* @param value1 first value, (between -(2^62)+1 and 2^62)
* @param value2 second value, (between -(2^62)+1 and 2^62)
* @param buffer
* @param offsetInBuffer
* @return number of bytes appended to the provided buffer
*/
public static int encodeInto(final long value1, final long value2, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer;
final int bytesAdded1 = encodeInto(value1, buffer, offset);
if (bytesAdded1 > 0) {
offset += bytesAdded1;
final int bytesAdded2 = encodeInto(value2, buffer, offset);
int offset = offsetInBuffer;
final int bytesAdded1 = encodeInto(value1, buffer, offset);
if (bytesAdded1 > 0) {
offset += bytesAdded1;
final int bytesAdded2 = encodeInto(value2, buffer, offset);
if (bytesAdded2 > 0) {
// both value fit into the buffer
// return the number of added bytes
return bytesAdded1 + bytesAdded2;
} else {
// second value did not fit into the buffer,
// remove the first value
// and return 0 to indicate that the values did not fit
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0;
}
}
if (bytesAdded2 > 0) {
// both value fit into the buffer
// return the number of added bytes
return bytesAdded1 + bytesAdded2;
} else {
// second value did not fit into the buffer,
// remove the first value
// and return 0 to indicate that the values did not fit
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0;
}
}
// return 0 if the encoded bytes do not fit
// the caller will have to provide a new buffer
return 0;
}
// return 0 if the encoded bytes do not fit
// the caller will have to provide a new buffer
return 0;
}
public static LongList decode(final byte[] buffer) {
public static LongList decode(final byte[] buffer) {
final LongList result = new LongList();
decodeInto(buffer, result);
return result;
}
final LongList result = new LongList();
decodeInto(buffer, result);
return result;
}
public static int encodeInto(final long value, final byte[] buffer, final int offsetInBuffer) {
public static int encodeInto(final long value, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer;
int offset = offsetInBuffer;
assert value >= MIN_VALUE : "min encodable value is -2^62+1 = " + MIN_VALUE;
assert value <= MAX_VALUE : "max encodable value is 2^62 = " + MAX_VALUE;
assert value >= MIN_VALUE : "min encodable value is -2^62+1 = " + MIN_VALUE;
assert value <= MAX_VALUE : "max encodable value is 2^62 = " + MAX_VALUE;
long normVal = encodeIntoPositiveValue(value);
long normVal = encodeIntoPositiveValue(value);
try {
final long maxFirstByteValue = 127;
try {
final long maxFirstByteValue = 127;
while (normVal > maxFirstByteValue) {
buffer[offset] = (byte) ((normVal & DATA_BITS) | CONTINUATION_BYTE_FLAG);
offset++;
normVal = normVal >> 7; // shift by number of value bits
}
buffer[offset] = (byte) (normVal);
return offset - offsetInBuffer + 1; // return number of encoded bytes
} catch (final ArrayIndexOutOfBoundsException e) {
// We need more bytes to store the value than are available.
// Reset the bytes we just wrote.
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0;
}
}
while (normVal > maxFirstByteValue) {
buffer[offset] = (byte) ((normVal & DATA_BITS) | CONTINUATION_BYTE_FLAG);
offset++;
normVal = normVal >> 7; // shift by number of value bits
}
buffer[offset] = (byte) (normVal);
return offset - offsetInBuffer + 1; // return number of encoded bytes
} catch (final ArrayIndexOutOfBoundsException e) {
// We need more bytes to store the value than are available.
// Reset the bytes we just wrote.
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0;
}
}
private static void decodeInto(final byte[] buffer, final LongList bufferedLongs) {
for (int i = 0; i < buffer.length; i++) {
private static void decodeInto(final byte[] buffer, final LongList bufferedLongs) {
for (int i = 0; i < buffer.length; i++) {
if (buffer[i] == 0) {
// no value is encoded to 0 => there are no further values
break;
} else {
long val = buffer[i] & DATA_BITS;
int shift = 7;
while (!isLastByte(buffer[i]) && i + 1 < buffer.length) {
val = val | ((buffer[i + 1] & DATA_BITS) << shift);
i++;
shift += 7;
}
bufferedLongs.add(decodeIntoSignedValue(val));
}
}
}
if (buffer[i] == 0) {
// no value is encoded to 0 => there are no further values
break;
} else {
long val = buffer[i] & DATA_BITS;
int shift = 7;
while (!isLastByte(buffer[i]) && i + 1 < buffer.length) {
val = val | ((buffer[i + 1] & DATA_BITS) << shift);
i++;
shift += 7;
}
bufferedLongs.add(decodeIntoSignedValue(val));
}
}
}
/**
* The input value (positive, negative or null) is encoded into a positive
* value.
*
* <pre>
*
* input: 0 1 -1 2 -2 3 -3
* encoded: 1 2 3 4 5 6 7
* </pre>
*/
private static long encodeIntoPositiveValue(final long value) {
return value > 0 ? value * 2 : (value * -2) + 1;
}
/**
* The input value (positive, negative or null) is encoded into a positive
* value.
*
* <pre>
*
* input: 0 1 -1 2 -2 3 -3
* encoded: 1 2 3 4 5 6 7
* </pre>
*/
private static long encodeIntoPositiveValue(final long value) {
return value > 0 ? value * 2 : (value * -2) + 1;
}
/**
* inverse of {@link #encodeIntoPositiveValue(long)}
*
* @param value
* @return
*/
private static long decodeIntoSignedValue(final long value) {
return (value / 2) * (value % 2 == 0 ? 1 : -1);
}
/**
* inverse of {@link #encodeIntoPositiveValue(long)}
*
* @param value
* @return
*/
private static long decodeIntoSignedValue(final long value) {
return (value / 2) * (value % 2 == 0 ? 1 : -1);
}
private static boolean isLastByte(final byte b) {
return (b & CONTINUATION_BYTE_FLAG) == 0;
}
private static boolean isLastByte(final byte b) {
return (b & CONTINUATION_BYTE_FLAG) == 0;
}
public static byte[] encode(final long... longs) {
public static byte[] encode(final long... longs) {
int neededBytes = 0;
for (final long l : longs) {
neededBytes += VariableByteEncoder.neededBytes(l);
}
int neededBytes = 0;
for (final long l : longs) {
neededBytes += VariableByteEncoder.neededBytes(l);
}
final byte[] result = new byte[neededBytes];
final byte[] result = new byte[neededBytes];
final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) {
throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
}
final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) {
throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
}
return result;
}
return result;
}
public static long decodeFirstValue(final byte[] buffer) {
public static long decodeFirstValue(final byte[] buffer) {
int offset = 0;
long val = buffer[offset] & DATA_BITS;
int shift = 7;
while (!isLastByte(buffer[offset]) && offset + 1 < buffer.length) {
val = val | ((buffer[offset + 1] & DATA_BITS) << shift);
offset++;
shift += 7;
}
return decodeIntoSignedValue(val);
}
int offset = 0;
long val = buffer[offset] & DATA_BITS;
int shift = 7;
while (!isLastByte(buffer[offset]) && offset + 1 < buffer.length) {
val = val | ((buffer[offset + 1] & DATA_BITS) << shift);
offset++;
shift += 7;
}
return decodeIntoSignedValue(val);
}
public static int encodeInto(final LongList values, final byte[] buffer, final int offsetInBuffer) {
public static int encodeInto(final LongList values, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer;
for (int i = 0; i < values.size(); i++) {
final long value = values.get(i);
int offset = offsetInBuffer;
for (int i = 0; i < values.size(); i++) {
final long value = values.get(i);
final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0;
}
offset += bytesAdded;
}
return offset - offsetInBuffer;
}
final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0;
}
offset += bytesAdded;
}
return offset - offsetInBuffer;
}
public static int encodeInto(final long[] values, final byte[] buffer, final int offsetInBuffer) {
public static int encodeInto(final long[] values, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer;
for (int i = 0; i < values.length; i++) {
final long value = values[i];
int offset = offsetInBuffer;
for (int i = 0; i < values.length; i++) {
final long value = values[i];
final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0;
}
offset += bytesAdded;
}
return offset - offsetInBuffer;
}
final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0;
}
offset += bytesAdded;
}
return offset - offsetInBuffer;
}
public static byte[] encode(final LongList longs) {
public static byte[] encode(final LongList longs) {
final int neededBytes = longs.stream().mapToInt(VariableByteEncoder::neededBytes).sum();
final byte[] result = new byte[neededBytes];
final int neededBytes = longs.stream().mapToInt(VariableByteEncoder::neededBytes).sum();
final byte[] result = new byte[neededBytes];
final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) {
throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
}
final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) {
throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
}
return result;
}
return result;
}
public static int neededBytes(final long value) {
final byte[] buffer = SINGLE_VALUE_BUFFER.get();
final int usedBytes = encodeInto(value, buffer, 0);
return usedBytes;
}
public static int neededBytes(final long value) {
final byte[] buffer = SINGLE_VALUE_BUFFER.get();
final int usedBytes = encodeInto(value, buffer, 0);
return usedBytes;
}
}

162
byte-utils/src/test/java/org/lucares/utils/byteencoder/VariableByteEncoderTest.java
View File

@@ -14,97 +14,97 @@ import org.testng.annotations.Test;
@Test
public class VariableByteEncoderTest {
@DataProvider
public Object[][] providerEncodeDecode() {
return new Object[][] { //
// encoded into 1 byte
{ 10, -5, 5 }, //
{ 10, 0, 5 }, //
{ 10, -63, 63 }, //
// encoded into 2 bytes
{ 10, 130, 131 }, //
// encoded into 3 bytes
{ 10, -8191, 8191 }, //
// encoded into n bytes
{ 1, Long.MAX_VALUE / 2 - 4, Long.MAX_VALUE / 2 }, //
{ 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2 }, //
{ 11, Long.MIN_VALUE / 2 + 1, Long.MIN_VALUE / 2 + 3 }, //
{ 12, Long.MAX_VALUE / 2 - 3, Long.MAX_VALUE / 2 },//
};
}
@DataProvider
public Object[][] providerEncodeDecode() {
return new Object[][] { //
// encoded into 1 byte
{ 10, -5, 5 }, //
{ 10, 0, 5 }, //
{ 10, -63, 63 }, //
// encoded into 2 bytes
{ 10, 130, 131 }, //
// encoded into 3 bytes
{ 10, -8191, 8191 }, //
// encoded into n bytes
{ 1, Long.MAX_VALUE / 2 - 4, Long.MAX_VALUE / 2 }, //
{ 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2 }, //
{ 11, Long.MIN_VALUE / 2 + 1, Long.MIN_VALUE / 2 + 3 }, //
{ 12, Long.MAX_VALUE / 2 - 3, Long.MAX_VALUE / 2 },//
};
}
@Test(dataProvider = "providerEncodeDecode")
public void testEncodeDecode(final long numValues, final long minValue, final long maxValue) {
@Test(dataProvider = "providerEncodeDecode")
public void testEncodeDecode(final long numValues, final long minValue, final long maxValue) {
final LongList originalValues = new LongList();
final byte[] buffer = new byte[1024];
final AtomicInteger offsetInBuffer = new AtomicInteger(0);
final LongList originalValues = new LongList();
final byte[] buffer = new byte[1024];
final AtomicInteger offsetInBuffer = new AtomicInteger(0);
ThreadLocalRandom.current().longs(numValues, minValue, maxValue).forEachOrdered(value -> {
originalValues.add(value);
final int appendedBytes = VariableByteEncoder.encodeInto(value, buffer, offsetInBuffer.get());
offsetInBuffer.addAndGet(appendedBytes);
});
ThreadLocalRandom.current().longs(numValues, minValue, maxValue).forEachOrdered(value -> {
originalValues.add(value);
final int appendedBytes = VariableByteEncoder.encodeInto(value, buffer, offsetInBuffer.get());
offsetInBuffer.addAndGet(appendedBytes);
});
final LongList actualValues = VariableByteEncoder.decode(buffer);
final LongList actualValues = VariableByteEncoder.decode(buffer);
assertEquals(actualValues.toString(), originalValues.toString());
}
assertEquals(actualValues.toString(), originalValues.toString());
}
@DataProvider
public Object[][] providerEncodeDecodeOfTwoValues() {
return new Object[][] { //
{ 12345, 67890, false, 1 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 2 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 3 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 4 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 5 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, true, 6 }, // both values need three bytes
{ 12345, 67890, true, 10 }, //
};
}
@DataProvider
public Object[][] providerEncodeDecodeOfTwoValues() {
return new Object[][] { //
{ 12345, 67890, false, 1 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 2 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 3 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 4 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 5 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, true, 6 }, // both values need three bytes
{ 12345, 67890, true, 10 }, //
};
}
@Test(dataProvider = "providerEncodeDecodeOfTwoValues")
public void testEncodeDecodeOfTwoValues(final long value1, final long value2, final boolean fits,
final int bufferSize) {
final LongList originalValues = new LongList();
final byte[] buffer = new byte[bufferSize];
@Test(dataProvider = "providerEncodeDecodeOfTwoValues")
public void testEncodeDecodeOfTwoValues(final long value1, final long value2, final boolean fits,
final int bufferSize) {
final LongList originalValues = new LongList();
final byte[] buffer = new byte[bufferSize];
final int bytesAdded = VariableByteEncoder.encodeInto(value1, value2, buffer, 0);
Assert.assertEquals(bytesAdded > 0, fits);
if (fits) {
originalValues.addAll(value1, value2);
} else {
Assert.assertEquals(buffer, new byte[bufferSize],
"checks that buffer is resetted after it discovers the values do not fit");
}
final int bytesAdded = VariableByteEncoder.encodeInto(value1, value2, buffer, 0);
Assert.assertEquals(bytesAdded > 0, fits);
if (fits) {
originalValues.addAll(value1, value2);
} else {
Assert.assertEquals(buffer, new byte[bufferSize],
"checks that buffer is resetted after it discovers the values do not fit");
}
final LongList decodedValues = VariableByteEncoder.decode(buffer);
Assert.assertEquals(decodedValues, originalValues);
}
final LongList decodedValues = VariableByteEncoder.decode(buffer);
Assert.assertEquals(decodedValues, originalValues);
}
@DataProvider
public Object[][] providerNededBytes() {
return new Object[][] { //
{ 0, 1 }, //
{ -10, 1 }, //
{ 10, 1 }, //
{ -63, 1 }, //
{ 63, 1 }, //
{ -64, 2 }, //
{ 64, 2 }, //
{ -8191, 2 }, //
{ 8191, 2 }, //
{ -8192, 3 }, //
{ 8192, 3 }, //
};
}
@DataProvider
public Object[][] providerNededBytes() {
return new Object[][] { //
{ 0, 1 }, //
{ -10, 1 }, //
{ 10, 1 }, //
{ -63, 1 }, //
{ 63, 1 }, //
{ -64, 2 }, //
{ 64, 2 }, //
{ -8191, 2 }, //
{ 8191, 2 }, //
{ -8192, 3 }, //
{ 8192, 3 }, //
};
}
@Test(dataProvider = "providerNededBytes")
public void testNeededBytes(final long value, final int expectedNeededBytes) {
@Test(dataProvider = "providerNededBytes")
public void testNeededBytes(final long value, final int expectedNeededBytes) {
final int neededBytes = VariableByteEncoder.neededBytes(value);
final byte[] encoded = VariableByteEncoder.encode(value);
Assert.assertEquals(encoded.length, neededBytes);
}
final int neededBytes = VariableByteEncoder.neededBytes(value);
final byte[] encoded = VariableByteEncoder.encode(value);
Assert.assertEquals(encoded.length, neededBytes);
}
}

94
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -5,60 +5,60 @@ import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.internal.ParititionId;
public class Doc {
private final Tags tags;
private final Tags tags;
/**
* the block number used by {@link BSFile}
*/
private final long rootBlockNumber;
/**
* the block number used by {@link BSFile}
*/
private final long rootBlockNumber;
private ParititionId partitionId;
private ParititionId partitionId;
/**
* Initializes a new document.
* <p>
* The path can be {@code null}. If path is {@code null}, then
* {@code offsetInListingFile} must be set. The path will be initialized lazily
* when needed.
* <p>
* This is used to reduce the memory footprint.
*
* @param tags
* @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath}
*/
public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) {
this.partitionId = partitionId;
this.tags = tags;
this.rootBlockNumber = rootBlockNumber;
}
/**
* Initializes a new document.
* <p>
* The path can be {@code null}. If path is {@code null}, then
* {@code offsetInListingFile} must be set. The path will be initialized lazily
* when needed.
* <p>
* This is used to reduce the memory footprint.
*
* @param tags
* @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath}
*/
public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) {
this.partitionId = partitionId;
this.tags = tags;
this.rootBlockNumber = rootBlockNumber;
}
public ParititionId getPartitionId() {
return partitionId;
}
public ParititionId getPartitionId() {
return partitionId;
}
public Tags getTags() {
return tags;
}
public Tags getTags() {
return tags;
}
/**
* the block number used by {@link BSFile}
*
* @return the root block number of this document
*/
public long getRootBlockNumber() {
return rootBlockNumber;
}
/**
* the block number used by {@link BSFile}
*
* @return the root block number of this document
*/
public long getRootBlockNumber() {
return rootBlockNumber;
}
public void setPartitionId(final ParititionId partitionId) {
this.partitionId = partitionId;
}
public void setPartitionId(final ParititionId partitionId) {
this.partitionId = partitionId;
}
@Override
public String toString() {
return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
@Override
public String toString() {
return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
}

8
data-store/src/main/java/org/lucares/pdb/datastore/InvalidValueException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class InvalidValueException extends IllegalArgumentException {
private static final long serialVersionUID = -8707541995666127297L;
private static final long serialVersionUID = -8707541995666127297L;
public InvalidValueException(final String msg) {
super(msg);
}
public InvalidValueException(final String msg) {
super(msg);
}
}

132
data-store/src/main/java/org/lucares/pdb/datastore/PdbFile.java
View File

@@ -14,86 +14,86 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class PdbFile {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private final PartitionDiskStore partitionDiskStorage;
private final PartitionDiskStore partitionDiskStorage;
public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) {
this.partitionDiskStorage = partitionDiskStorage;
}
public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) {
this.partitionDiskStorage = partitionDiskStorage;
}
@Override
public Stream<LongList> apply(final PdbFile pdbFile) {
final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId());
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
return bsFile.streamOfLongLists();
}
}
@Override
public Stream<LongList> apply(final PdbFile pdbFile) {
final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId());
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
return bsFile.streamOfLongLists();
}
}
private final Tags tags;
private final Tags tags;
/**
* The rootBlockNumber to be used by {@link BSFile}
*/
private final long rootBlockNumber;
/**
* The rootBlockNumber to be used by {@link BSFile}
*/
private final long rootBlockNumber;
private final ParititionId partitionId;
private final ParititionId partitionId;
public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) {
this.partitionId = partitionId;
this.rootBlockNumber = rootBlockNumber;
this.tags = tags;
}
public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) {
this.partitionId = partitionId;
this.rootBlockNumber = rootBlockNumber;
this.tags = tags;
}
public Tags getTags() {
return tags;
}
public Tags getTags() {
return tags;
}
public long getRootBlockNumber() {
return rootBlockNumber;
}
public long getRootBlockNumber() {
return rootBlockNumber;
}
public ParititionId getPartitionId() {
return partitionId;
}
public ParititionId getPartitionId() {
return partitionId;
}
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) {
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) {
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
return longStream;
}
return longStream;
}
@Override
public String toString() {
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId="+partitionId+"]";
}
@Override
public String toString() {
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId=" + partitionId + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
}

172
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -1,105 +1,105 @@
package org.lucares.pdb.datastore;
public class Proposal implements Comparable<Proposal> {
private final String proposedTag;
private final String proposedTag;
private final String proposedQuery;
private final String proposedQuery;
private final boolean hasResults;
private final boolean hasResults;
private final String newQuery;
private final String newQuery;
private final int newCaretPosition;
private final int newCaretPosition;
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults,
final String newQuery, final int newCaretPosition) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.hasResults = hasResults;
this.newQuery = newQuery;
this.newCaretPosition = newCaretPosition;
}
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults,
final String newQuery, final int newCaretPosition) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.hasResults = hasResults;
this.newQuery = newQuery;
this.newCaretPosition = newCaretPosition;
}
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.hasResults = hasResults;
this.newQuery = proposal.newQuery;
this.newCaretPosition = proposal.newCaretPosition;
}
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.hasResults = hasResults;
this.newQuery = proposal.newQuery;
this.newCaretPosition = proposal.newCaretPosition;
}
public String getProposedTag() {
return proposedTag;
}
public String getProposedTag() {
return proposedTag;
}
public String getProposedQuery() {
return proposedQuery;
}
public String getProposedQuery() {
return proposedQuery;
}
public boolean hasResults() {
return hasResults;
}
public boolean hasResults() {
return hasResults;
}
public String getNewQuery() {
return newQuery;
}
public String getNewQuery() {
return newQuery;
}
public int getNewCaretPosition() {
return newCaretPosition;
}
public int getNewCaretPosition() {
return newCaretPosition;
}
@Override
public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults="
+ hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]";
}
@Override
public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults="
+ hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + newCaretPosition;
result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode());
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + newCaretPosition;
result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode());
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (newCaretPosition != other.newCaretPosition)
return false;
if (newQuery == null) {
if (other.newQuery != null)
return false;
} else if (!newQuery.equals(other.newQuery))
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
} else if (!proposedQuery.equals(other.proposedQuery))
return false;
if (proposedTag == null) {
if (other.proposedTag != null)
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (newCaretPosition != other.newCaretPosition)
return false;
if (newQuery == null) {
if (other.newQuery != null)
return false;
} else if (!newQuery.equals(other.newQuery))
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
} else if (!proposedQuery.equals(other.proposedQuery))
return false;
if (proposedTag == null) {
if (other.proposedTag != null)
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
return true;
}
@Override
public int compareTo(final Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
@Override
public int compareTo(final Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
}

8
data-store/src/main/java/org/lucares/pdb/datastore/ReadException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class ReadException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public ReadException(final RuntimeException e) {
super(e);
}
public ReadException(final RuntimeException e) {
super(e);
}
}

20
data-store/src/main/java/org/lucares/pdb/datastore/ReadRuntimeException.java
View File

@@ -2,17 +2,17 @@ package org.lucares.pdb.datastore;
public class ReadRuntimeException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause);
}
public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause);
}
public ReadRuntimeException(final String message) {
super(message);
}
public ReadRuntimeException(final String message) {
super(message);
}
public ReadRuntimeException(final Throwable cause) {
super(cause);
}
public ReadRuntimeException(final Throwable cause) {
super(cause);
}
}

14
data-store/src/main/java/org/lucares/pdb/datastore/WriteException.java
View File

@@ -2,14 +2,14 @@ package org.lucares.pdb.datastore;
public class WriteException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public WriteException(final String message, final Throwable cause) {
super(message, cause);
}
public WriteException(final String message, final Throwable cause) {
super(message, cause);
}
public WriteException(final Throwable cause) {
super(cause);
}
public WriteException(final Throwable cause) {
super(cause);
}
}

670
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -39,381 +39,381 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore implements AutoCloseable {
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
public static final char LISTING_FILE_SEPARATOR = ',';
public static final char LISTING_FILE_SEPARATOR = ',';
public static final String SUBDIR_STORAGE = "storage";
public static final String SUBDIR_STORAGE = "storage";
// used to generate doc ids that are
// a) unique
// b) monotonically increasing (this is, so that we don't have to sort the doc
// ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
// used to generate doc ids that are
// a) unique
// b) monotonically increasing (this is, so that we don't have to sort the doc
// ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
public static Tag TAG_ALL_DOCS = null;
public static Tag TAG_ALL_DOCS = null;
private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc;
private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc;
private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId;
private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId;
private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId;
private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId;
private final QueryCompletionIndex queryCompletionIndex;
private final QueryCompletionIndex queryCompletionIndex;
// A Doc will never be changed once it is created. Therefore we can cache them
// easily.
private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000);
// A Doc will never be changed once it is created. Therefore we can cache them
// easily.
private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000);
private final HotEntryCache<Tags, PdbWriter> writerCache;
private final HotEntryCache<Tags, PdbWriter> writerCache;
private final PartitionDiskStore diskStorage;
private final Path storageBasePath;
private final PartitionDiskStore diskStorage;
private final Path storageBasePath;
public DataStore(final Path dataDirectory) throws IOException {
storageBasePath = storageDirectory(dataDirectory);
public DataStore(final Path dataDirectory) throws IOException {
storageBasePath = storageDirectory(dataDirectory);
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it
// must be initialized after the string compressor has been created
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it
// must be initialized after the string compressor has been created
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs",
new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs",
new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(),
PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(),
PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER,
new DocEncoderDecoder());
docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER,
new DocEncoderDecoder());
queryCompletionIndex = new QueryCompletionIndex(storageBasePath);
queryCompletionIndex = new QueryCompletionIndex(storageBasePath);
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000);
writerCache.addListener((key, value) -> value.close());
}
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000);
writerCache.addListener((key, value) -> value.close());
}
private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv");
}
private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv");
}
public static Path storageDirectory(final Path dataDirectory) throws IOException {
return dataDirectory.resolve(SUBDIR_STORAGE);
}
public static Path storageDirectory(final Path dataDirectory) throws IOException {
return dataDirectory.resolve(SUBDIR_STORAGE);
}
public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags);
writer.write(dateAsEpochMilli, value);
}
public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags);
writer.write(dateAsEpochMilli, value);
}
// visible for test
QueryCompletionIndex getQueryCompletionIndex() {
return queryCompletionIndex;
}
// visible for test
QueryCompletionIndex getQueryCompletionIndex() {
return queryCompletionIndex;
}
public long createNewFile(final ParititionId partitionId, final Tags tags) {
try {
final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
public long createNewFile(final ParititionId partitionId, final Tags tags) {
try {
final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
final long docId = createUniqueDocId();
final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset);
docIdToDoc.putValue(partitionId, docId, doc);
final long docId = createUniqueDocId();
final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset);
docIdToDoc.putValue(partitionId, docId, doc);
final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId);
Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags);
final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId);
Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags);
// store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS);
for (final Tag tag : ts) {
// store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS);
for (final Tag tag : ts) {
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
if (diskStoreOffsetForDocIdsOfTag == null) {
diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag);
}
if (diskStoreOffsetForDocIdsOfTag == null) {
diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag);
}
try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag,
partitionId)) {
docIdsOfTag.append(docId);
}
}
try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag,
partitionId)) {
docIdsOfTag.append(docId);
}
}
// index the tags, so that we can efficiently find all possible values for a
// field in a query
queryCompletionIndex.addTags(partitionId, tags);
// index the tags, so that we can efficiently find all possible values for a
// field in a query
queryCompletionIndex.addTags(partitionId, tags);
return newFilesRootBlockOffset;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
return newFilesRootBlockOffset;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private long createUniqueDocId() {
return NEXT_DOC_ID.getAndIncrement();
}
private long createUniqueDocId() {
return NEXT_DOC_ID.getAndIncrement();
}
public List<PdbFile> getFilesForQuery(final Query query) {
public List<PdbFile> getFilesForQuery(final Query query) {
final List<Doc> searchResult = search(query);
if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results.");
}
final List<Doc> searchResult = search(query);
if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results.");
}
final List<PdbFile> result = toPdbFiles(searchResult);
return result;
}
final List<PdbFile> result = toPdbFiles(searchResult);
return result;
}
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) {
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) {
final ParititionId partitionId = document.getPartitionId();
final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags);
final ParititionId partitionId = document.getPartitionId();
final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags);
result.add(pdbFile);
}
return result;
}
result.add(pdbFile);
}
return result;
}
public List<Doc> search(final Query query) {
try {
final List<Doc> result = new ArrayList<>();
public List<Doc> search(final Query query) {
try {
final List<Doc> result = new ArrayList<>();
final PartitionLongList docIdsList = executeQuery(query);
LOGGER.trace("query {} found {} docs", query, docIdsList.size());
final List<Doc> docs = mapDocIdsToDocs(docIdsList);
result.addAll(docs);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public int count(final Query query) {
final PartitionLongList docIdsList = executeQuery(query);
return docIdsList.size();
}
public List<String> getAvailableFields(final DateTimeRange dateRange) {
final Set<String> keys = new HashSet<>();
final Tag keyPrefix = new Tag("", ""); // will find everything
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString()));
keys.remove(ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys);
Collections.sort(result);
return result;
}
private PartitionLongList executeQuery(final Query query) {
final long start = System.nanoTime();
synchronized (docIdToDoc) {
final Expression expression = QueryLanguageParser.parse(query.getQuery());
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId,
diskStorage);
final PartitionLongList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query,
(System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}
}
private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException {
final List<Doc> result = new ArrayList<>(docIdsList.size());
synchronized (docIdToDoc) {
final long start = System.nanoTime();
for (final ParititionId partitionId : docIdsList) {
final LongList docIds = docIdsList.get(partitionId);
for (int i = 0; i < docIds.size(); i++) {
final long docId = docIds.get(i);
final Doc doc = getDocByDocId(partitionId, docId);
Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist.");
result.add(doc);
}
}
MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(),
(System.nanoTime() - start) / 1_000_000.0);
}
return result;
}
public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) {
final Long docId = tagsToDocId.getValue(partitionId, tags);
if (docId != null) {
final Doc doc = getDocByDocId(partitionId, docId);
return Optional.of(doc);
}
return Optional.empty();
}
public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) {
final List<Doc> result = new ArrayList<>();
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags);
for (final Long docId : docIds) {
if (docId != null) {
final Doc doc = getDocByDocId(dateRange, docId);
result.add(doc);
}
}
return result;
}
private Doc getDocByDocId(final ParititionId partitionId, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
return docIdToDoc.getValue(partitionId, documentId);
});
}
private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId);
if (docIds.size() == 1) {
return docIds.get(0);
} else if (docIds.size() > 1) {
throw new IllegalStateException(
"Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds);
}
throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId);
});
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex);
final List<Proposal> proposals = newProposerParser.propose(query);
LOGGER.debug("Proposals for query {}: {}", query, proposals);
return proposals;
}
public PartitionDiskStore getDiskStorage() {
return diskStorage;
}
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags));
}
// visible for test
long sizeWriterCache() {
return writerCache.size();
}
private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) {
final Optional<Doc> docsForTags = getByTags(partitionId, tags);
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
} catch (final RuntimeException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(partitionId, tags);
}
return writer;
}
private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) {
final long rootBlockNumber = createNewFile(partitionId, tags);
final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags);
return result;
}
@Override
public void close() throws RuntimeIOException {
try {
// we cannot simply clear the cache, because the cache implementation (Guava at
// the time of writing) handles eviction events asynchronously.
forEachWriter(cachedWriter -> {
try {
cachedWriter.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
} finally {
try {
diskStorage.close();
} finally {
tagToDocsId.close();
}
}
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> {
try {
consumer.accept(writer);
} catch (final RuntimeException e) {
LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
final PartitionLongList docIdsList = executeQuery(query);
LOGGER.trace("query {} found {} docs", query, docIdsList.size());
final List<Doc> docs = mapDocIdsToDocs(docIdsList);
result.addAll(docs);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public int count(final Query query) {
final PartitionLongList docIdsList = executeQuery(query);
return docIdsList.size();
}
public List<String> getAvailableFields(final DateTimeRange dateRange) {
final Set<String> keys = new HashSet<>();
final Tag keyPrefix = new Tag("", ""); // will find everything
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString()));
keys.remove(ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys);
Collections.sort(result);
return result;
}
private PartitionLongList executeQuery(final Query query) {
final long start = System.nanoTime();
synchronized (docIdToDoc) {
final Expression expression = QueryLanguageParser.parse(query.getQuery());
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId,
diskStorage);
final PartitionLongList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query,
(System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}
}
private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException {
final List<Doc> result = new ArrayList<>(docIdsList.size());
synchronized (docIdToDoc) {
final long start = System.nanoTime();
for (final ParititionId partitionId : docIdsList) {
final LongList docIds = docIdsList.get(partitionId);
for (int i = 0; i < docIds.size(); i++) {
final long docId = docIds.get(i);
final Doc doc = getDocByDocId(partitionId, docId);
Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist.");
result.add(doc);
}
}
MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(),
(System.nanoTime() - start) / 1_000_000.0);
}
return result;
}
public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) {
final Long docId = tagsToDocId.getValue(partitionId, tags);
if (docId != null) {
final Doc doc = getDocByDocId(partitionId, docId);
return Optional.of(doc);
}
return Optional.empty();
}
public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) {
final List<Doc> result = new ArrayList<>();
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags);
for (final Long docId : docIds) {
if (docId != null) {
final Doc doc = getDocByDocId(dateRange, docId);
result.add(doc);
}
}
return result;
}
private Doc getDocByDocId(final ParititionId partitionId, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
return docIdToDoc.getValue(partitionId, documentId);
});
}
private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId);
if (docIds.size() == 1) {
return docIds.get(0);
} else if (docIds.size() > 1) {
throw new IllegalStateException(
"Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds);
}
throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId);
});
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex);
final List<Proposal> proposals = newProposerParser.propose(query);
LOGGER.debug("Proposals for query {}: {}", query, proposals);
return proposals;
}
public PartitionDiskStore getDiskStorage() {
return diskStorage;
}
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags));
}
// visible for test
long sizeWriterCache() {
return writerCache.size();
}
private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) {
final Optional<Doc> docsForTags = getByTags(partitionId, tags);
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
} catch (final RuntimeException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(partitionId, tags);
}
return writer;
}
private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) {
final long rootBlockNumber = createNewFile(partitionId, tags);
final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags);
return result;
}
@Override
public void close() throws RuntimeIOException {
try {
// we cannot simply clear the cache, because the cache implementation (Guava at
// the time of writing) handles eviction events asynchronously.
forEachWriter(cachedWriter -> {
try {
cachedWriter.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
} finally {
try {
diskStorage.close();
} finally {
tagToDocsId.close();
}
}
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> {
try {
consumer.accept(writer);
} catch (final RuntimeException e) {
LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
}

268
data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java
View File

@@ -19,178 +19,178 @@ import org.lucares.pdb.api.DateTimeRange;
public class DateIndexExtension {
/**
* This date pattern defines the resolution of the date index
*/
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
/**
* This date pattern defines the resolution of the date index
*/
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
// visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
// visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null);
private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null);
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>();
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>();
OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) {
OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) {
result.add(toDateIndexPrefix(current));
current = current.plusMonths(1);
result.add(toDateIndexPrefix(current));
current = current.plusMonths(1);
}
result.add(toDateIndexPrefix(dateRange.getEnd()));
}
result.add(toDateIndexPrefix(dateRange.getEnd()));
return result;
}
return result;
}
static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN);
}
static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN);
}
public static ParititionId toPartitionId(final long epochMilli) {
String result;
final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get();
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
&& lastAccessed.getMaxEpochMilli() >= epochMilli) {
result = lastAccessed.getDatePrefix();
} else {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
public static ParititionId toPartitionId(final long epochMilli) {
String result;
final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get();
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
&& lastAccessed.getMaxEpochMilli() >= epochMilli) {
result = lastAccessed.getDatePrefix();
} else {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
LAST_ACCESSED.set(newValue);
} else {
result = value.getValue().getDatePrefix();
LAST_ACCESSED.set(value.getValue());
}
}
return new ParititionId(result);
}
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
LAST_ACCESSED.set(newValue);
} else {
result = value.getValue().getDatePrefix();
LAST_ACCESSED.set(value.getValue());
}
}
return new ParititionId(result);
}
public static String toDateIndexPrefix(final long epochMilli) {
public static String toDateIndexPrefix(final long epochMilli) {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
String result;
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
} else {
result = value.getValue().getDatePrefix();
}
String result;
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
} else {
result = value.getValue().getDatePrefix();
}
return result;
}
return result;
}
/**
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final
* Collection<? extends PartitionId> availablePartitionIds) instead
*
* @param dateRange
* @return
*/
static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) {
final List<ParititionId> result = new ArrayList<>();
/**
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final
* Collection<? extends PartitionId> availablePartitionIds) instead
*
* @param dateRange
* @return
*/
static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) {
final List<ParititionId> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
while (!current.isAfter(end)) {
final String id = current.format(DATE_PATTERN);
final ParititionId partitionId = new ParititionId(id);
result.add(partitionId);
current = current.plusMonths(1);
}
while (!current.isAfter(end)) {
final String id = current.format(DATE_PATTERN);
final ParititionId partitionId = new ParititionId(id);
result.add(partitionId);
current = current.plusMonths(1);
}
return result;
}
return result;
}
public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange,
final Collection<? extends ParititionId> availablePartitionIds) {
final Set<ParititionId> result = new LinkedHashSet<>();
public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange,
final Collection<? extends ParititionId> availablePartitionIds) {
final Set<ParititionId> result = new LinkedHashSet<>();
final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli());
final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli());
final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli());
final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli());
for (final ParititionId partitionId : availablePartitionIds) {
if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) {
result.add(partitionId);
}
}
for (final ParititionId partitionId : availablePartitionIds) {
if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) {
result.add(partitionId);
}
}
return result;
}
return result;
}
public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
}
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
}
public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) {
final List<Long> result = new ArrayList<>();
public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) {
final List<Long> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
while (!current.isAfter(end)) {
result.add(current.toInstant().toEpochMilli());
current = current.plusMonths(1);
}
while (!current.isAfter(end)) {
result.add(current.toInstant().toEpochMilli());
current = current.plusMonths(1);
}
return result;
}
return result;
}
public static ParititionId now() {
return toPartitionId(System.currentTimeMillis());
}
public static ParititionId now() {
return toPartitionId(System.currentTimeMillis());
}
}
class DatePrefixAndRange {
private final String datePrefix;
private final long minEpochMilli;
private final long maxEpochMilli;
private final String datePrefix;
private final long minEpochMilli;
private final long maxEpochMilli;
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super();
this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli;
}
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super();
this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli;
}
public String getDatePrefix() {
return datePrefix;
}
public String getDatePrefix() {
return datePrefix;
}
public long getMinEpochMilli() {
return minEpochMilli;
}
public long getMinEpochMilli() {
return minEpochMilli;
}
public long getMaxEpochMilli() {
return maxEpochMilli;
}
public long getMaxEpochMilli() {
return maxEpochMilli;
}
public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
}
public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
}
@Override
public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
}
@Override
public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
}
}

16
data-store/src/main/java/org/lucares/pdb/datastore/internal/DatePartitioner.java
View File

@@ -6,14 +6,14 @@ import org.lucares.pdb.api.DateTimeRange;
public class DatePartitioner implements PartitionIdSource {
private final DateTimeRange dateRange;
private final DateTimeRange dateRange;
public DatePartitioner(final DateTimeRange dateRange) {
this.dateRange = dateRange;
}
public DatePartitioner(final DateTimeRange dateRange) {
this.dateRange = dateRange;
}
@Override
public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) {
return DateIndexExtension.toPartitionIds(dateRange, availablePartitions);
}
@Override
public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) {
return DateIndexExtension.toPartitionIds(dateRange, availablePartitions);
}
}

62
data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java
View File

@@ -8,43 +8,43 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
@Override
public byte[] encode(final Doc doc) {
@Override
public byte[] encode(final Doc doc) {
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().toBytes();
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().toBytes();
final byte[] result = new byte[rootBlockNumber.length + tags.length];
final byte[] result = new byte[rootBlockNumber.length + tags.length];
System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length);
System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length);
System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length);
System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length);
return result;
}
return result;
}
@Override
public Doc decode(final byte[] bytes) {
@Override
public Doc decode(final byte[] bytes) {
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(null, tags, rootBlockNumber);
}
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(null, tags, rootBlockNumber);
}
@Override
public Doc encodeValue(final Doc v) {
return v;
}
@Override
public Doc encodeValue(final Doc v) {
return v;
}
@Override
public Doc decodeValue(final ParititionId partitionId, final Doc t) {
if (t != null) {
t.setPartitionId(partitionId);
}
return t;
}
public byte[] getEmptyValue() {
return new byte[] {0};
}
@Override
public Doc decodeValue(final ParititionId partitionId, final Doc t) {
if (t != null) {
t.setPartitionId(partitionId);
}
return t;
}
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}

22
data-store/src/main/java/org/lucares/pdb/datastore/internal/GlobMatcher.java
View File

@@ -7,18 +7,18 @@ import org.lucares.pdb.datastore.lang.GloblikePattern;
public class GlobMatcher {
private final Pattern pattern;
private final Pattern pattern;
public GlobMatcher(final String globlike) {
pattern = GloblikePattern.globlikeToRegex(globlike);
}
public GlobMatcher(final String globlike) {
pattern = GloblikePattern.globlikeToRegex(globlike);
}
public GlobMatcher(final Iterable<String> globlikes) {
pattern = GloblikePattern.globlikeToRegex(globlikes);
}
public GlobMatcher(final Iterable<String> globlikes) {
pattern = GloblikePattern.globlikeToRegex(globlikes);
}
public boolean matches(final String s) {
final Matcher matcher = pattern.matcher(s);
return matcher.find();
}
public boolean matches(final String s) {
final Matcher matcher = pattern.matcher(s);
return matcher.find();
}
}

106
data-store/src/main/java/org/lucares/pdb/datastore/internal/ParititionId.java
View File

@@ -1,65 +1,65 @@
package org.lucares.pdb.datastore.internal;
public class ParititionId implements Comparable<ParititionId> {
private final String partitionId;
private final String partitionId;
/**
* Create a new partition id.
*
* @param partitionId the id, e.g. a time like 201902 (partition for entries of
* February 2019)
*/
public ParititionId(final String partitionId) {
super();
this.partitionId = partitionId;
}
/**
* Create a new partition id.
*
* @param partitionId the id, e.g. a time like 201902 (partition for entries of
* February 2019)
*/
public ParititionId(final String partitionId) {
super();
this.partitionId = partitionId;
}
public static ParititionId of(final String partitionId) {
return new ParititionId(partitionId);
}
public static ParititionId of(final String partitionId) {
return new ParititionId(partitionId);
}
@Override
public int compareTo(final ParititionId other) {
return partitionId.compareTo(other.getPartitionId());
}
@Override
public int compareTo(final ParititionId other) {
return partitionId.compareTo(other.getPartitionId());
}
/**
* @return the id, e.g. a time like 201902 (partition for entries of February
* 2019)
*/
public String getPartitionId() {
return partitionId;
}
/**
* @return the id, e.g. a time like 201902 (partition for entries of February
* 2019)
*/
public String getPartitionId() {
return partitionId;
}
@Override
public String toString() {
return partitionId;
}
@Override
public String toString() {
return partitionId;
}
/*
* non-standard hashcode implementation! This class is just a wrapper for
* string, so we delegate directly to String.hashCode().
*/
@Override
public int hashCode() {
return partitionId.hashCode();
}
/*
* non-standard hashcode implementation! This class is just a wrapper for
* string, so we delegate directly to String.hashCode().
*/
@Override
public int hashCode() {
return partitionId.hashCode();
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ParititionId other = (ParititionId) obj;
if (partitionId == null) {
if (other.partitionId != null)
return false;
} else if (!partitionId.equals(other.partitionId))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ParititionId other = (ParititionId) obj;
if (partitionId == null) {
if (other.partitionId != null)
return false;
} else if (!partitionId.equals(other.partitionId))
return false;
return true;
}
}

4
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareEncoderDecoder.java
View File

@@ -4,7 +4,7 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public interface PartitionAwareEncoderDecoder<V, P> extends EncoderDecoder<P> {
public P encodeValue(V v);
public P encodeValue(V v);
public V decodeValue(ParititionId partitionId, P p);
public V decodeValue(ParititionId partitionId, P p);
}

54
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareWrapper.java
View File

@@ -4,37 +4,37 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public final class PartitionAwareWrapper<O> implements PartitionAwareEncoderDecoder<O, O> {
private final EncoderDecoder<O> delegate;
private final EncoderDecoder<O> delegate;
public PartitionAwareWrapper(final EncoderDecoder<O> delegate) {
this.delegate = delegate;
}
public PartitionAwareWrapper(final EncoderDecoder<O> delegate) {
this.delegate = delegate;
}
@Override
public byte[] encode(final O object) {
return delegate.encode(object);
}
@Override
public byte[] encode(final O object) {
return delegate.encode(object);
}
@Override
public O decode(final byte[] bytes) {
return delegate.decode(bytes);
}
@Override
public O decode(final byte[] bytes) {
return delegate.decode(bytes);
}
@Override
public O encodeValue(final O v) {
return v;
}
@Override
public O encodeValue(final O v) {
return v;
}
@Override
public O decodeValue(final ParititionId partitionId, final O p) {
return p;
}
@Override
public O decodeValue(final ParititionId partitionId, final O p) {
return p;
}
public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) {
return new PartitionAwareWrapper<>(encoder);
}
public byte[] getEmptyValue() {
return delegate.getEmptyValue();
}
public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) {
return new PartitionAwareWrapper<>(encoder);
}
public byte[] getEmptyValue() {
return delegate.getEmptyValue();
}
}

108
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionDiskStore.java
View File

@@ -14,68 +14,68 @@ import org.lucares.pdb.blockstorage.LongStreamFile;
import org.lucares.pdb.diskstorage.DiskStorage;
public class PartitionDiskStore {
private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>();
private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>();
private final Function<ParititionId, DiskStorage> creator;
private final Function<ParititionId, DiskStorage> supplier;
private final Function<ParititionId, DiskStorage> creator;
private final Function<ParititionId, DiskStorage> supplier;
public PartitionDiskStore(final Path storageBasePath, final String filename) {
public PartitionDiskStore(final Path storageBasePath, final String filename) {
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
final boolean isNew = !Files.exists(file);
final DiskStorage diskStorage = new DiskStorage(file, storageBasePath);
if (isNew) {
diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE);
}
return diskStorage;
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new DiskStorage(file, storageBasePath);
}
return null;
};
}
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
final boolean isNew = !Files.exists(file);
final DiskStorage diskStorage = new DiskStorage(file, storageBasePath);
if (isNew) {
diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE);
}
return diskStorage;
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new DiskStorage(file, storageBasePath);
}
return null;
};
}
public DiskStorage getExisting(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, supplier);
}
public DiskStorage getExisting(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, supplier);
}
public DiskStorage getCreateIfNotExists(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, creator);
}
public DiskStorage getCreateIfNotExists(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, creator);
}
public long allocateBlock(final ParititionId partitionId, final int blockSize) {
final DiskStorage diskStorage = getCreateIfNotExists(partitionId);
return diskStorage.allocateBlock(blockSize);
}
public long allocateBlock(final ParititionId partitionId, final int blockSize) {
final DiskStorage diskStorage = getCreateIfNotExists(partitionId);
return diskStorage.allocateBlock(blockSize);
}
public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) {
try {
final DiskStorage diskStorage = getExisting(partitionId);
return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) {
try {
final DiskStorage diskStorage = getExisting(partitionId);
return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public void close() {
final List<Throwable> throwables = new ArrayList<>();
public void close() {
final List<Throwable> throwables = new ArrayList<>();
for (final DiskStorage diskStorage : diskStorages.values()) {
try {
diskStorage.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
for (final DiskStorage diskStorage : diskStorages.values()) {
try {
diskStorage.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
}
}

2
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionIdSource.java
View File

@@ -3,5 +3,5 @@ package org.lucares.pdb.datastore.internal;
import java.util.Set;
public interface PartitionIdSource {
Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions);
Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions);
}

140
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionLongList.java
View File

@@ -9,87 +9,87 @@ import java.util.Set;
import org.lucares.collections.LongList;
public class PartitionLongList implements Iterable<ParititionId> {
private final Map<ParititionId, LongList> lists = new HashMap<>();
private final Map<ParititionId, LongList> lists = new HashMap<>();
public LongList put(final ParititionId partitionId, final LongList longList) {
return lists.put(partitionId, longList);
}
public LongList put(final ParititionId partitionId, final LongList longList) {
return lists.put(partitionId, longList);
}
public LongList get(final ParititionId partitionId) {
return lists.get(partitionId);
}
public LongList get(final ParititionId partitionId) {
return lists.get(partitionId);
}
@Override
public Iterator<ParititionId> iterator() {
return lists.keySet().iterator();
}
@Override
public Iterator<ParititionId> iterator() {
return lists.keySet().iterator();
}
public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
if (x != null && y != null) {
final LongList intersection = LongList.intersection(x, y);
result.put(partitionId, intersection);
} else {
// one list is empty => the intersection is empty
}
}
return result;
}
if (x != null && y != null) {
final LongList intersection = LongList.intersection(x, y);
result.put(partitionId, intersection);
} else {
// one list is empty => the intersection is empty
}
}
return result;
}
public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
if (x != null && y != null) {
final LongList intersection = LongList.union(x, y);
result.put(partitionId, intersection);
} else if (x != null) {
result.put(partitionId, x.clone());
} else if (y != null) {
result.put(partitionId, y.clone());
}
}
return result;
}
if (x != null && y != null) {
final LongList intersection = LongList.union(x, y);
result.put(partitionId, intersection);
} else if (x != null) {
result.put(partitionId, x.clone());
} else if (y != null) {
result.put(partitionId, y.clone());
}
}
return result;
}
public int size() {
int size = 0;
public int size() {
int size = 0;
for (final LongList longList : lists.values()) {
size += longList.size();
}
for (final LongList longList : lists.values()) {
size += longList.size();
}
return size;
}
return size;
}
public boolean isSorted() {
for (final LongList longList : lists.values()) {
if (!longList.isSorted()) {
return false;
}
}
return true;
}
public boolean isSorted() {
for (final LongList longList : lists.values()) {
if (!longList.isSorted()) {
return false;
}
}
return true;
}
public void removeAll(final PartitionLongList remove) {
for (final ParititionId partitionId : lists.keySet()) {
final LongList removeLongList = remove.get(partitionId);
if (removeLongList != null) {
lists.get(partitionId).removeAll(removeLongList);
}
}
}
public void removeAll(final PartitionLongList remove) {
for (final ParititionId partitionId : lists.keySet()) {
final LongList removeLongList = remove.get(partitionId);
if (removeLongList != null) {
lists.get(partitionId).removeAll(removeLongList);
}
}
}
}

212
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java
View File

@@ -25,130 +25,130 @@ import org.lucares.pdb.map.Visitor;
*/
public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final Function<ParititionId, PersistentMap<K, P>> creator;
private final Function<ParititionId, PersistentMap<K, P>> supplier;
private final Function<ParititionId, PersistentMap<K, P>> creator;
private final Function<ParititionId, PersistentMap<K, P>> supplier;
private final PartitionAwareEncoderDecoder<V, P> valueEncoder;
private final PartitionAwareEncoderDecoder<V, P> valueEncoder;
public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder,
final PartitionAwareEncoderDecoder<V, P> valueEncoder) {
public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder,
final PartitionAwareEncoderDecoder<V, P> valueEncoder) {
this.valueEncoder = valueEncoder;
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
}
return null;
};
preload(storageBasePath);
}
this.valueEncoder = valueEncoder;
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
}
return null;
};
preload(storageBasePath);
}
private void preload(final Path storageBasePath) {
try {
Files.list(storageBasePath)//
.filter(Files::isDirectory)//
.map(Path::getFileName)//
.map(Path::toString)//
.map(ParititionId::of)//
.forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier));
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private void preload(final Path storageBasePath) {
try {
Files.list(storageBasePath)//
.filter(Files::isDirectory)//
.map(Path::getFileName)//
.map(Path::toString)//
.map(ParititionId::of)//
.forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier));
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private Set<ParititionId> getAllPartitionIds() {
return maps.keySet();
}
private Set<ParititionId> getAllPartitionIds() {
return maps.keySet();
}
public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) {
return partitionIdSource.toPartitionIds(getAllPartitionIds());
}
public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) {
return partitionIdSource.toPartitionIds(getAllPartitionIds());
}
private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, supplier);
}
private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, supplier);
}
private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, creator);
}
private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, creator);
}
public V getValue(final ParititionId partitionId, final K key) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
final P persistedValue = map != null ? map.getValue(key) : null;
return valueEncoder.decodeValue(partitionId, persistedValue);
}
public V getValue(final ParititionId partitionId, final K key) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
final P persistedValue = map != null ? map.getValue(key) : null;
return valueEncoder.decodeValue(partitionId, persistedValue);
}
public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) {
final List<V> result = new ArrayList<>();
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) {
final List<V> result = new ArrayList<>();
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
if (map != null) {
final V value = valueEncoder.decodeValue(partitionId, map.getValue(key));
if (value != null) {
result.add(value);
}
}
}
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
if (map != null) {
final V value = valueEncoder.decodeValue(partitionId, map.getValue(key));
if (value != null) {
result.add(value);
}
}
}
return result;
}
return result;
}
public V putValue(final ParititionId partitionId, final K key, final V value) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
final P persistedValue = valueEncoder.encodeValue(value);
final P previousPersistedValue = map.putValue(key, persistedValue);
return valueEncoder.decodeValue(partitionId, previousPersistedValue);
}
public V putValue(final ParititionId partitionId, final K key, final V value) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
final P persistedValue = valueEncoder.encodeValue(value);
final P previousPersistedValue = map.putValue(key, persistedValue);
return valueEncoder.decodeValue(partitionId, previousPersistedValue);
}
public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) {
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) {
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
}
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
}
@Override
public void close() {
final List<Throwable> throwables = new ArrayList<>();
@Override
public void close() {
final List<Throwable> throwables = new ArrayList<>();
for (final PersistentMap<K, P> map : maps.values()) {
try {
map.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
for (final PersistentMap<K, P> map : maps.values()) {
try {
map.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
}

88
data-store/src/main/java/org/lucares/pdb/datastore/internal/PdbWriter.java
View File

@@ -17,62 +17,62 @@ import org.slf4j.LoggerFactory;
*/
class PdbWriter implements AutoCloseable, Flushable {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
private final PdbFile pdbFile;
private long lastEpochMilli;
private final PdbFile pdbFile;
private long lastEpochMilli;
private final TimeSeriesFile timeSeriesFile;
private final TimeSeriesFile timeSeriesFile;
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) {
this.pdbFile = pdbFile;
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) {
this.pdbFile = pdbFile;
timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct?
timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct?
lastEpochMilli = optionalLastValue.orElse(0L);
}
lastEpochMilli = optionalLastValue.orElse(0L);
}
public PdbFile getPdbFile() {
return pdbFile;
}
public PdbFile getPdbFile() {
return pdbFile;
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException {
try {
timeSeriesFile.appendTimeValue(epochMilli, value);
public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException {
try {
timeSeriesFile.appendTimeValue(epochMilli, value);
lastEpochMilli = epochMilli;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
lastEpochMilli = epochMilli;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
@Override
public void close() {
@Override
public void close() {
LOGGER.debug("close PdbWriter {}", pdbFile);
timeSeriesFile.close();
}
LOGGER.debug("close PdbWriter {}", pdbFile);
timeSeriesFile.close();
}
@Override
public void flush() {
timeSeriesFile.flush();
}
@Override
public void flush() {
timeSeriesFile.flush();
}
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) {
writer.write(entry.getEpochMilli(), entry.getValue());
}
}
}
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) {
writer.write(entry.getEpochMilli(), entry.getValue());
}
}
}
@Override
public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";
}
@Override
public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";
}
}

506
data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java
View File

@@ -143,321 +143,321 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*
*/
public class QueryCompletionIndex implements AutoCloseable {
private static final class TwoTags {
private final Tag tagA;
private final Tag tagB;
private static final class TwoTags {
private final Tag tagA;
private final Tag tagB;
public TwoTags(final Tag tagA, final Tag tagB) {
this.tagA = tagA;
this.tagB = tagB;
}
public TwoTags(final Tag tagA, final Tag tagB) {
this.tagA = tagA;
this.tagB = tagB;
}
public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) {
public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) {
tagA = new Tag(fieldA, valueA);
tagB = new Tag(fieldB, valueB);
}
tagA = new Tag(fieldA, valueA);
tagB = new Tag(fieldB, valueB);
}
public Tag getTagA() {
return tagA;
}
public Tag getTagA() {
return tagA;
}
public Tag getTagB() {
return tagB;
}
public Tag getTagB() {
return tagB;
}
@Override
public String toString() {
return tagA + "::" + tagB;
}
}
@Override
public String toString() {
return tagA + "::" + tagB;
}
}
public static final class FieldField {
private final int fieldA;
private final int fieldB;
public static final class FieldField {
private final int fieldA;
private final int fieldB;
public FieldField(final int fieldA, final int fieldB) {
this.fieldA = fieldA;
this.fieldB = fieldB;
}
public FieldField(final int fieldA, final int fieldB) {
this.fieldA = fieldA;
this.fieldB = fieldB;
}
public int getFieldA() {
return fieldA;
}
public int getFieldA() {
return fieldA;
}
public int getFieldB() {
return fieldB;
}
public int getFieldB() {
return fieldB;
}
@Override
public String toString() {
return fieldA + "::" + fieldB;
}
}
@Override
public String toString() {
return fieldA + "::" + fieldB;
}
}
private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> {
private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> {
@Override
public byte[] encode(final TwoTags tagAndField) {
final LongList tmp = new LongList(4);
final Tag tagA = tagAndField.getTagA();
final Tag tagB = tagAndField.getTagB();
@Override
public byte[] encode(final TwoTags tagAndField) {
final LongList tmp = new LongList(4);
final Tag tagA = tagAndField.getTagA();
final Tag tagB = tagAndField.getTagB();
tmp.add(tagB.getKey());
tmp.add(tagA.getKey());
tmp.add(tagB.getKey());
tmp.add(tagA.getKey());
if (tagA.getValue() >= 0) {
tmp.add(tagA.getValue());
if (tagA.getValue() >= 0) {
tmp.add(tagA.getValue());
// A query for tagA.key and tagA.value and tagB.key is done by setting
// tagB.value==-1.
// The query is then executed as a prefix search. Thus tagB.value must not be
// part of the byte array that is returned.
if (tagB.getValue() >= 0) {
tmp.add(tagB.getValue());
}
} else {
Preconditions.checkSmaller(tagB.getValue(), 0,
"if no value for tagA is given, then tagB must also be empty");
}
// A query for tagA.key and tagA.value and tagB.key is done by setting
// tagB.value==-1.
// The query is then executed as a prefix search. Thus tagB.value must not be
// part of the byte array that is returned.
if (tagB.getValue() >= 0) {
tmp.add(tagB.getValue());
}
} else {
Preconditions.checkSmaller(tagB.getValue(), 0,
"if no value for tagA is given, then tagB must also be empty");
}
return VariableByteEncoder.encode(tmp);
}
return VariableByteEncoder.encode(tmp);
}
@Override
public TwoTags decode(final byte[] bytes) {
@Override
public TwoTags decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int tagBKey = (int) tmp.get(0);
final int tagAKey = (int) tmp.get(1);
final int tagAValue = (int) tmp.get(2);
final int tagBValue = (int) tmp.get(3);
final LongList tmp = VariableByteEncoder.decode(bytes);
final int tagBKey = (int) tmp.get(0);
final int tagAKey = (int) tmp.get(1);
final int tagAValue = (int) tmp.get(2);
final int tagBValue = (int) tmp.get(3);
final Tag tagA = new Tag(tagAKey, tagAValue);
final Tag tagB = new Tag(tagBKey, tagBValue);
final Tag tagA = new Tag(tagAKey, tagAValue);
final Tag tagB = new Tag(tagBKey, tagBValue);
return new TwoTags(tagA, tagB);
}
return new TwoTags(tagA, tagB);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0, 0, 0, 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0, 0, 0, 0 };
}
}
private static final class EncoderTag implements EncoderDecoder<Tag> {
private static final class EncoderTag implements EncoderDecoder<Tag> {
@Override
public byte[] encode(final Tag tag) {
@Override
public byte[] encode(final Tag tag) {
final LongList longList = new LongList(2);
longList.add(tag.getKey());
final LongList longList = new LongList(2);
longList.add(tag.getKey());
if (tag.getValue() >= 0) {
longList.add(tag.getValue());
}
return VariableByteEncoder.encode(longList);
}
if (tag.getValue() >= 0) {
longList.add(tag.getValue());
}
return VariableByteEncoder.encode(longList);
}
@Override
public Tag decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int key = (int) tmp.get(0);
final int value = (int) tmp.get(1);
return new Tag(key, value);
}
@Override
public Tag decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int key = (int) tmp.get(0);
final int value = (int) tmp.get(1);
return new Tag(key, value);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private static final class EncoderField implements EncoderDecoder<String> {
private static final class EncoderField implements EncoderDecoder<String> {
@Override
public byte[] encode(final String field) {
@Override
public byte[] encode(final String field) {
if (field.isEmpty()) {
return new byte[0];
}
if (field.isEmpty()) {
return new byte[0];
}
return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field));
}
return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field));
}
@Override
public String decode(final byte[] bytes) {
final long compressedString = VariableByteEncoder.decodeFirstValue(bytes);
return Tags.STRING_COMPRESSOR.get((int) compressedString);
}
@Override
public String decode(final byte[] bytes) {
final long compressedString = VariableByteEncoder.decodeFirstValue(bytes);
return Tags.STRING_COMPRESSOR.get((int) compressedString);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex;
private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex;
private final PartitionPersistentMap<String, Empty, Empty> fieldIndex;
private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex;
private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex;
private final PartitionPersistentMap<String, Empty, Empty> fieldIndex;
public QueryCompletionIndex(final Path basePath) throws IOException {
tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
public QueryCompletionIndex(final Path basePath) throws IOException {
tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs",
new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs",
new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
}
fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
}
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags();
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags();
// index all combinations of tagA and tagB and fieldA to fieldB
for (final Tag tagA : listOfTagsA) {
for (final Tag tagB : listOfTagsB) {
final TwoTags key = new TwoTags(tagA, tagB);
tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE);
}
}
// index all combinations of tagA and tagB and fieldA to fieldB
for (final Tag tagA : listOfTagsA) {
for (final Tag tagB : listOfTagsB) {
final TwoTags key = new TwoTags(tagA, tagB);
tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE);
}
}
// create indices of all tags and all fields
for (final Tag tag : listOfTagsA) {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE);
}
}
// create indices of all tags and all fields
for (final Tag tag : listOfTagsA) {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE);
}
}
@Override
public void close() throws IOException {
tagToTagIndex.close();
}
@Override
public void close() throws IOException {
tagToTagIndex.close();
}
/**
* Find values for fieldB that are yield results when executing the query
* "fieldA=valueA and fieldB=???"
*
* @param dateRange the date range
* @param fieldA the other field of the and expression
* @param valueA {@link GlobMatcher} for the value of the other field
* @param fieldB the field we are searching values for
* @return values of fieldB
*/
public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA,
final String fieldB) {
/**
* Find values for fieldB that are yield results when executing the query
* "fieldA=valueA and fieldB=???"
*
* @param dateRange the date range
* @param fieldA the other field of the and expression
* @param valueA {@link GlobMatcher} for the value of the other field
* @param fieldB the field we are searching values for
* @return values of fieldB
*/
public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA,
final String fieldB) {
final SortedSet<String> result = new TreeSet<>();
final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null);
final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final String vA = k.getTagA().getValueAsString();
final String vA = k.getTagA().getValueAsString();
if (valueA.matches(vA)) {
result.add(k.getTagB().getValueAsString());
}
});
if (valueA.matches(vA)) {
result.add(k.getTagB().getValueAsString());
}
});
return result;
}
return result;
}
/**
* Find values for fieldB that are yield results when executing the query
* "tag.field=tag.value and fieldB=???"
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return values for the field
*/
public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) {
/**
* Find values for fieldB that are yield results when executing the query
* "tag.field=tag.value and fieldB=???"
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return values for the field
*/
public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) {
final SortedSet<String> result = new TreeSet<>();
final int tagBKey = Tags.STRING_COMPRESSOR.put(field);
final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See
// EncoderTwoTags
final TwoTags keyPrefix = new TwoTags(tag, tagB);
final SortedSet<String> result = new TreeSet<>();
final int tagBKey = Tags.STRING_COMPRESSOR.put(field);
final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See
// EncoderTwoTags
final TwoTags keyPrefix = new TwoTags(tag, tagB);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getTagB().getValueAsString());
});
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getTagB().getValueAsString());
});
return result;
}
return result;
}
/**
* Find all values for the given field.
*
* @param dateRange the date range
* @param field the field
* @return the values
*/
public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) {
/**
* Find all values for the given field.
*
* @param dateRange the date range
* @param field the field
* @return the values
*/
public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) {
final SortedSet<String> result = new TreeSet<>();
final int tagKey = Tags.STRING_COMPRESSOR.put(field);
final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See
final SortedSet<String> result = new TreeSet<>();
final int tagKey = Tags.STRING_COMPRESSOR.put(field);
final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getValueAsString());
});
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getValueAsString());
});
return result;
}
return result;
}
/**
* Find values for {@code field} that will yield results for the query
* "tag.field=tag.value and not field=???".
* <p>
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return the values
*/
public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag,
final String field) {
final SortedSet<String> result = new TreeSet<>();
/**
* Find values for {@code field} that will yield results for the query
* "tag.field=tag.value and not field=???".
* <p>
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return the values
*/
public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag,
final String field) {
final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null);
final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null);
final int negatedValueA = tag.getValue();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final int negatedValueA = tag.getValue();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final int valueA = k.getTagA().getValue();
if (valueA != negatedValueA) {
result.add(k.getTagB().getValueAsString());
}
});
final int valueA = k.getTagA().getValue();
if (valueA != negatedValueA) {
result.add(k.getTagB().getValueAsString());
}
});
return result;
}
return result;
}
public SortedSet<String> findAllFields(final DateTimeRange dateRange) {
final SortedSet<String> result = new TreeSet<>();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
result.add(k);
});
return result;
}
public SortedSet<String> findAllFields(final DateTimeRange dateRange) {
final SortedSet<String> result = new TreeSet<>();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
result.add(k);
});
return result;
}
public boolean hasField(final DateTimeRange dateRange, final String field) {
final AtomicBoolean found = new AtomicBoolean(false);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
if (k.equals(field)) {
found.set(true);
}
});
return found.get();
}
public boolean hasField(final DateTimeRange dateRange, final String field) {
final AtomicBoolean found = new AtomicBoolean(false);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
if (k.equals(field)) {
found.set(true);
}
});
return found.get();
}
}

92
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagEncoderDecoder.java
View File

@@ -7,58 +7,58 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
import org.lucares.utils.byteencoder.VariableByteEncoder;
class TagEncoderDecoder implements EncoderDecoder<Tag> {
@Override
public byte[] encode(final Tag tag) {
@Override
public byte[] encode(final Tag tag) {
final LongList keyAndValueCompressed = new LongList(2);
final LongList keyAndValueCompressed = new LongList(2);
final String key = tag.getKeyAsString();
final byte[] result;
if (!key.isEmpty()) {
final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key);
keyAndValueCompressed.add(keyAsLong);
final String key = tag.getKeyAsString();
final byte[] result;
if (!key.isEmpty()) {
final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key);
keyAndValueCompressed.add(keyAsLong);
final String value = tag.getValueAsString();
if (!value.isEmpty()) {
final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value);
keyAndValueCompressed.add(valueAsLong);
}
result = VariableByteEncoder.encode(keyAndValueCompressed);
} else {
result = new byte[0];
}
final String value = tag.getValueAsString();
if (!value.isEmpty()) {
final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value);
keyAndValueCompressed.add(valueAsLong);
}
result = VariableByteEncoder.encode(keyAndValueCompressed);
} else {
result = new byte[0];
}
return result;
}
return result;
}
@Override
public Tag decode(final byte[] bytes) {
final LongList compressedStrings = VariableByteEncoder.decode(bytes);
final Tag result;
switch (compressedStrings.size()) {
case 0:
@Override
public Tag decode(final byte[] bytes) {
final LongList compressedStrings = VariableByteEncoder.decode(bytes);
final Tag result;
switch (compressedStrings.size()) {
case 0:
result = new Tag("", "");
break;
case 1:
final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
result = new Tag(k, "");
result = new Tag("", "");
break;
case 1:
final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
result = new Tag(k, "");
break;
case 2:
final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1));
result = new Tag(key, value);
break;
default:
throw new IllegalStateException("too many values: " + compressedStrings);
}
break;
case 2:
final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1));
result = new Tag(key, value);
break;
default:
throw new IllegalStateException("too many values: " + compressedStrings);
}
return result;
}
@Override
public byte[] getEmptyValue() {
return new byte[] {0};
}
return result;
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}

26
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagsEncoderDecoder.java
View File

@@ -4,18 +4,18 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
class TagsEncoderDecoder implements EncoderDecoder<Tags> {
@Override
public byte[] encode(final Tags tags) {
return tags.toBytes();
}
@Override
public byte[] encode(final Tags tags) {
return tags.toBytes();
}
@Override
public Tags decode(final byte[] bytes) {
return Tags.fromBytes(bytes);
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
@Override
public Tags decode(final byte[] bytes) {
return Tags.fromBytes(bytes);
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
}

64
data-store/src/main/java/org/lucares/pdb/datastore/lang/CandidateGrouper.java
View File

@@ -9,43 +9,43 @@ import java.util.TreeSet;
import java.util.regex.Pattern;
public class CandidateGrouper {
public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) {
public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) {
final TreeSet<String> result = new TreeSet<>();
final int numDotsInValue = countDotsInValue(queryWithCaretMarker);
final TreeSet<String> result = new TreeSet<>();
final int numDotsInValue = countDotsInValue(queryWithCaretMarker);
for (final String value : values) {
// keep everything up to the (numDotsInValue+1)-th
final String[] token = value.split(Pattern.quote("."));
final List<String> tokenlist = new ArrayList<>(Arrays.asList(token));
final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1);
String shortenedValue = String.join(".", prefix);
if (tokenlist.size() > numDotsInValue + 1) {
shortenedValue += ".";
}
result.add(shortenedValue);
}
for (final String value : values) {
// keep everything up to the (numDotsInValue+1)-th
final String[] token = value.split(Pattern.quote("."));
final List<String> tokenlist = new ArrayList<>(Arrays.asList(token));
final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1);
String shortenedValue = String.join(".", prefix);
if (tokenlist.size() > numDotsInValue + 1) {
shortenedValue += ".";
}
result.add(shortenedValue);
}
return result;
}
return result;
}
private int countDotsInValue(final String queryWithCaretMarker) {
private int countDotsInValue(final String queryWithCaretMarker) {
int count = 0;
int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1;
final String delimiter = " (),=!";
int count = 0;
int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1;
final String delimiter = " (),=!";
while (index >= 0) {
final char c = queryWithCaretMarker.charAt(index);
if (delimiter.indexOf(c) >= 0) {
break;
}
if (c == '.') {
count++;
}
index--;
}
while (index >= 0) {
final char c = queryWithCaretMarker.charAt(index);
if (delimiter.indexOf(c) >= 0) {
break;
}
if (c == '.') {
count++;
}
index--;
}
return count;
}
return count;
}
}

18
data-store/src/main/java/org/lucares/pdb/datastore/lang/ErrorListener.java
View File

@@ -6,14 +6,14 @@ import org.antlr.v4.runtime.Recognizer;
public class ErrorListener extends BaseErrorListener {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
final int lineStart = line;
final int startIndex = charPositionInLine;
final int lineStop = line;
final int stopIndex = charPositionInLine;
throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex);
}
final int lineStart = line;
final int startIndex = charPositionInLine;
final int lineStop = line;
final int stopIndex = charPositionInLine;
throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex);
}
}

1426
data-store/src/main/java/org/lucares/pdb/datastore/lang/Expression.java
View File

File diff suppressed because it is too large Load Diff

263
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java
View File

@@ -26,179 +26,180 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ExpressionToDocIdVisitor extends ExpressionVisitor<PartitionLongList> {
private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class);
private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class);
private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId;
private final PartitionDiskStore diskStorage;
private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId;
private final PartitionDiskStore diskStorage;
private final DatePartitioner datePartitioner;
private final DatePartitioner datePartitioner;
public ExpressionToDocIdVisitor(final DateTimeRange dateRange,
final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) {
this.datePartitioner = new DatePartitioner(dateRange);
this.keyToValueToDocId = keyToValueToDocsId;
this.diskStorage = diskStorage;
}
public ExpressionToDocIdVisitor(final DateTimeRange dateRange,
final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) {
this.datePartitioner = new DatePartitioner(dateRange);
this.keyToValueToDocId = keyToValueToDocsId;
this.diskStorage = diskStorage;
}
@Override
public PartitionLongList visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public PartitionLongList visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles);
LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles);
LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public PartitionLongList visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles);
LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles);
LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Not expression) {
@Override
public PartitionLongList visit(final Not expression) {
final Expression negatedExpression = expression.getExpression();
final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this);
final long start = System.nanoTime();
final Expression negatedExpression = expression.getExpression();
final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
result.removeAll(docIdsToBeNegated);
final PartitionLongList result = getAllDocIds();
result.removeAll(docIdsToBeNegated);
LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Parentheses parentheses) {
@Override
public PartitionLongList visit(final Parentheses parentheses) {
throw new UnsupportedOperationException(
"Parenthesis not supported. The correct order should come from the parser.");
}
throw new UnsupportedOperationException(
"Parenthesis not supported. The correct order should come from the parser.");
}
@Override
public PartitionLongList visit(final Expression.MatchAll expression) {
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
@Override
public PartitionLongList visit(final Expression.MatchAll expression) {
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
@Override
public PartitionLongList visit(final Expression.InExpression expression) {
final long start = System.nanoTime();
@Override
public PartitionLongList visit(final Expression.InExpression expression) {
final long start = System.nanoTime();
final String propertyName = expression.getProperty();
final List<String> values = expression.getValues();
final String propertyName = expression.getProperty();
final List<String> values = expression.getValues();
PartitionLongList result = new PartitionLongList();
PartitionLongList result = new PartitionLongList();
for (final String value : values) {
for (final String value : values) {
final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value));
result = PartitionLongList.union(result, docIds);
}
final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value));
result = PartitionLongList.union(result, docIds);
}
LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
private PartitionLongList getAllDocIds() {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
private PartitionLongList getAllDocIds() {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);
final LongList tmp = bsFile.asLongList();
result.put(partitionId, tmp);
}
}
return result;
}
if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);
final LongList tmp = bsFile.asLongList();
result.put(partitionId, tmp);
}
}
return result;
}
private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) {
final PartitionLongList result = new PartitionLongList();
private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) {
final PartitionLongList result = new PartitionLongList();
final long start = System.nanoTime();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final List<LongList> docIdsForPartition = new ArrayList<>();
keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> {
if (valuePattern.matcher(tags.getValueAsString()).matches()) {
try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds, partitionId)) {
final long start = System.nanoTime();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final List<LongList> docIdsForPartition = new ArrayList<>();
keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> {
if (valuePattern.matcher(tags.getValueAsString()).matches()) {
try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds,
partitionId)) {
// We know that all LongLists coming from a BSFile are sorted, non-overlapping
// and increasing, that means we can just concatenate them and get a sorted
// list.
final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList());
final LongList concatenatedLists = concatenateLists(longLists);
// We know that all LongLists coming from a BSFile are sorted, non-overlapping
// and increasing, that means we can just concatenate them and get a sorted
// list.
final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList());
final LongList concatenatedLists = concatenateLists(longLists);
Preconditions.checkTrue(concatenatedLists.isSorted(),
"The LongLists containing document ids must be sorted, "
+ "non-overlapping and increasing, so that the concatenation "
+ "is sorted. This is guaranteed by the fact that document ids "
+ "are generated in monotonically increasing order.");
Preconditions.checkTrue(concatenatedLists.isSorted(),
"The LongLists containing document ids must be sorted, "
+ "non-overlapping and increasing, so that the concatenation "
+ "is sorted. This is guaranteed by the fact that document ids "
+ "are generated in monotonically increasing order.");
docIdsForPartition.add(concatenatedLists);
}
}
});
docIdsForPartition.add(concatenatedLists);
}
}
});
final LongList mergedDocsIdsForPartition = merge(docIdsForPartition);
result.put(partitionId, mergedDocsIdsForPartition);
}
final LongList mergedDocsIdsForPartition = merge(docIdsForPartition);
result.put(partitionId, mergedDocsIdsForPartition);
}
LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0);
LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
return result;
}
private LongList merge(final Collection<LongList> lists) {
private LongList merge(final Collection<LongList> lists) {
LongList result = new LongList();
LongList result = new LongList();
for (final LongList list : lists) {
result = LongList.union(result, list);
}
for (final LongList list : lists) {
result = LongList.union(result, list);
}
return result;
}
return result;
}
private static LongList concatenateLists(final Collection<LongList> lists) {
private static LongList concatenateLists(final Collection<LongList> lists) {
final int totalSize = lists.stream().mapToInt(LongList::size).sum();
final LongList result = new LongList(totalSize);
final int totalSize = lists.stream().mapToInt(LongList::size).sum();
final LongList result = new LongList(totalSize);
for (final LongList list : lists) {
result.addAll(list);
}
for (final LongList list : lists) {
result.addAll(list);
}
return result;
return result;
}
}
}

66
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionVisitor.java
View File

@@ -1,47 +1,47 @@
package org.lucares.pdb.datastore.lang;
public abstract class ExpressionVisitor<T> {
public T visit(final Expression.And expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.And expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.InExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.InExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndNotCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndNotCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.CaretAndExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.CaretAndExpression expression) {
throw new UnsupportedOperationException();
}
}

436
data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java
View File

@@ -22,278 +22,278 @@ import org.slf4j.LoggerFactory;
public class FindValuesForQueryCompletion extends ExpressionVisitor<SortedSet<String>> {
private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret");
private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret");
private static final Logger METRIC_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation");
private static final Logger METRIC_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation");
private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> {
private final QueryCompletionIndex index;
private final String field;
private final DateTimeRange dateTimeRange;
private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> {
private final QueryCompletionIndex index;
private final String field;
private final DateTimeRange dateTimeRange;
public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange,
final QueryCompletionIndex queryCompletionIndex, final String field) {
this.dateTimeRange = dateTimeRange;
index = queryCompletionIndex;
this.field = field;
}
public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange,
final QueryCompletionIndex queryCompletionIndex, final String field) {
this.dateTimeRange = dateTimeRange;
index = queryCompletionIndex;
this.field = field;
}
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final SortedSet<String> result;
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = property.getField();
final String valueA = property.getValue().getValue();
final String fieldA = property.getField();
final String valueA = property.getValue().getValue();
final boolean hasField = index.hasField(dateTimeRange, fieldA);
if (hasField) {
final boolean hasField = index.hasField(dateTimeRange, fieldA);
if (hasField) {
final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA);
final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new);
final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA);
final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new);
final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size();
final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was
// chosen
// arbitrarily
if (useMultiFetch) {
result = new TreeSet<>();
final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size();
final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was
// chosen
// arbitrarily
if (useMultiFetch) {
result = new TreeSet<>();
for (final String v : valuesA) {
final Tag tagA = new Tag(fieldA, v);
final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field);
result.addAll(tmp);
}
} else {
result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field);
}
for (final String v : valuesA) {
final Tag tagA = new Tag(fieldA, v);
final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field);
result.addAll(tmp);
}
} else {
result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field);
}
METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)",
useMultiFetch ? "multi-fetch" : "single-fetch", property, field,
(System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100);
METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)",
useMultiFetch ? "multi-fetch" : "single-fetch", property, field,
(System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100);
} else {
result = new TreeSet<>();
}
return result;
}
} else {
result = new TreeSet<>();
}
return result;
}
@Override
public SortedSet<String> visit(final InExpression expression) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = expression.getProperty();
final List<String> values = expression.getValues();
@Override
public SortedSet<String> visit(final InExpression expression) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = expression.getProperty();
final List<String> values = expression.getValues();
result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field);
result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
try {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
try {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left instanceof Property && right instanceof Not) {
final Property leftProperty = (Property) left;
if (left instanceof Property && right instanceof Not) {
final Property leftProperty = (Property) left;
final SortedSet<String> allValuesForField = leftProperty.visit(this);
final SortedSet<String> allValuesForField = leftProperty.visit(this);
final Expression rightInnerExpression = ((Not) right).getExpression();
final SortedSet<String> rightResult = rightInnerExpression.visit(this);
final Expression rightInnerExpression = ((Not) right).getExpression();
final SortedSet<String> rightResult = rightInnerExpression.visit(this);
return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new);
return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new);
} else {
} else {
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult);
result.retainAll(rightResult);
return result;
}
} finally {
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
}
}
return result;
}
} finally {
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
}
}
@Override
public SortedSet<String> visit(final Or expression) {
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult);
result.addAll(rightResult);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Not expression) {
@Override
public SortedSet<String> visit(final Not expression) {
final long start = System.nanoTime();
if (!(expression.getExpression() instanceof Property)) {
throw new UnsupportedOperationException("NOT expressions like '" + expression
+ "' are not supported. Only 'NOT property=value' expressions are supported.");
}
final long start = System.nanoTime();
if (!(expression.getExpression() instanceof Property)) {
throw new UnsupportedOperationException("NOT expressions like '" + expression
+ "' are not supported. Only 'NOT property=value' expressions are supported.");
}
final Property property = (Property) expression.getExpression();
final Tag tag = new Tag(property.getField(), property.getValueAsString());
final Property property = (Property) expression.getExpression();
final Tag tag = new Tag(property.getField(), property.getValueAsString());
final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field);
final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field);
final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField,
valuesNotForField, TreeSet::new);
final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field);
final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field);
final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField,
valuesNotForField, TreeSet::new);
final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField,
TreeSet::new);
final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField,
TreeSet::new);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}
private final QueryCompletionIndex queryCompletionIndex;
private final DateTimeRange dateRange;
public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final QueryCompletionIndex queryCompletionIndex) {
this.dateRange = dateRange;
this.queryCompletionIndex = queryCompletionIndex;
}
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final String field = property.getField();
final String value = property.getValue().getValue();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valuePrefix;
if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
private final QueryCompletionIndex queryCompletionIndex;
@Override
public SortedSet<String> visit(final AndCaretExpression expression) {
private final DateTimeRange dateRange;
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final QueryCompletionIndex queryCompletionIndex) {
this.dateRange = dateRange;
this.queryCompletionIndex = queryCompletionIndex;
}
final Expression rightHandExpression = expression.getExpression();
@Override
public SortedSet<String> visit(final Property property) {
final SortedSet<String> candidateValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
final long start = System.nanoTime();
final String field = property.getField();
final String value = property.getValue().getValue();
final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new);
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final String valuePrefix;
@Override
public SortedSet<String> visit(final AndNotCaretExpression expression) {
if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
}
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange,
caretExpression.getField());
final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
valuePattern, TreeSet::new);
@Override
public SortedSet<String> visit(final AndCaretExpression expression) {
final Expression rightHandExpression = expression.getExpression();
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final SortedSet<String> rightHandValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
final Expression rightHandExpression = expression.getExpression();
if (rightHandValues.size() == 1) {
// there is only one alternative and that one must not be chosen
return Collections.emptySortedSet();
}
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> candidateValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
@Override
public SortedSet<String> visit(final Not expression) {
final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new);
final String field;
final Expression innerExpression = expression.getExpression();
if (innerExpression instanceof Property) {
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public SortedSet<String> visit(final AndNotCaretExpression expression) {
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
result.addAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange,
caretExpression.getField());
final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
valuePattern, TreeSet::new);
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression rightHandExpression = expression.getExpression();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> rightHandValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
result.retainAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
if (rightHandValues.size() == 1) {
// there is only one alternative and that one must not be chosen
return Collections.emptySortedSet();
}
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Not expression) {
final String field;
final Expression innerExpression = expression.getExpression();
if (innerExpression instanceof Property) {
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}

94
data-store/src/main/java/org/lucares/pdb/datastore/lang/GloblikePattern.java
View File

@@ -12,70 +12,70 @@ import org.slf4j.LoggerFactory;
public class GloblikePattern {
private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class);
enum FilterMode {
KEEP_EQUAL
}
enum FilterMode {
KEEP_EQUAL
}
public static Pattern globlikeToRegex(final String globlike) {
public static Pattern globlikeToRegex(final String globlike) {
final String valueRegex = "^" + globlikeToPattern(globlike);
final String valueRegex = "^" + globlikeToPattern(globlike);
LOGGER.trace(">{}< -> >{}<", globlike, valueRegex);
LOGGER.trace(">{}< -> >{}<", globlike, valueRegex);
return Pattern.compile(valueRegex);
}
return Pattern.compile(valueRegex);
}
public static Pattern globlikeToRegex(final Iterable<String> globlikes) {
public static Pattern globlikeToRegex(final Iterable<String> globlikes) {
final List<String> regex = new ArrayList<>();
final List<String> regex = new ArrayList<>();
for (final String globlike : globlikes) {
regex.add(globlikeToPattern(globlike));
}
final StringBuilder fullRegex = new StringBuilder("^(");
fullRegex.append(String.join("|", regex));
fullRegex.append(")");
for (final String globlike : globlikes) {
regex.add(globlikeToPattern(globlike));
}
final StringBuilder fullRegex = new StringBuilder("^(");
fullRegex.append(String.join("|", regex));
fullRegex.append(")");
LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex);
LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex);
return Pattern.compile(fullRegex.toString());
}
return Pattern.compile(fullRegex.toString());
}
private static String globlikeToPattern(final String globlike) {
// a character that cannot be in the globPattern
final String dotPlaceholder = "\ue003"; // fourth character in the private use area
private static String globlikeToPattern(final String globlike) {
// a character that cannot be in the globPattern
final String dotPlaceholder = "\ue003"; // fourth character in the private use area
final String valueRegex = globlike//
.replace("-", Pattern.quote("-"))//
.replace(".", dotPlaceholder)//
.replace("*", ".*")//
.replace(dotPlaceholder, ".*\\.")//
.replaceAll("([A-Z])", "[a-z]*$1");
return valueRegex;
}
final String valueRegex = globlike//
.replace("-", Pattern.quote("-"))//
.replace(".", dotPlaceholder)//
.replace("*", ".*")//
.replace(dotPlaceholder, ".*\\.")//
.replaceAll("([A-Z])", "[a-z]*$1");
return valueRegex;
}
public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues,
final String valuePattern, final Supplier<T> generator) {
final T result = generator.get();
public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues,
final String valuePattern, final Supplier<T> generator) {
final T result = generator.get();
return filterValues(result, availableValues, valuePattern);
}
return filterValues(result, availableValues, valuePattern);
}
public static <T extends Collection<String>> T filterValues(final T result,
final Collection<String> availableValues, final String valuePattern) {
public static <T extends Collection<String>> T filterValues(final T result,
final Collection<String> availableValues, final String valuePattern) {
final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern);
final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern);
for (final String value : availableValues) {
final Matcher matcher = pattern.matcher(value);
if (matcher.find()) {
result.add(value);
}
}
for (final String value : availableValues) {
final Matcher matcher = pattern.matcher(value);
if (matcher.find()) {
result.add(value);
}
}
return result;
}
return result;
}
}

96
data-store/src/main/java/org/lucares/pdb/datastore/lang/IdentityExpressionVisitor.java
View File

@@ -14,66 +14,66 @@ import org.lucares.pdb.datastore.lang.Expression.Property;
* as base class for visitors that modify expressions.
*/
public abstract class IdentityExpressionVisitor extends ExpressionVisitor<Expression> {
@Override
public Expression visit(final And expression) {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
return new And(left, right);
}
return new And(left, right);
}
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
return new Or(left, right);
}
return new Or(left, right);
}
@Override
public Expression visit(final Not expression) {
return new Not(expression.getExpression().visit(this));
}
@Override
public Expression visit(final Not expression) {
return new Not(expression.getExpression().visit(this));
}
@Override
public Expression visit(final Property expression) {
return expression;
}
@Override
public Expression visit(final Property expression) {
return expression;
}
@Override
public Expression visit(final Expression.Terminal expression) {
return expression;
}
@Override
public Expression visit(final Expression.Terminal expression) {
return expression;
}
@Override
public Expression visit(final Expression.MatchAll expression) {
return expression;
}
@Override
public Expression visit(final Expression.MatchAll expression) {
return expression;
}
@Override
public Expression visit(final Expression.InExpression expression) {
return expression;
}
@Override
public Expression visit(final Expression.InExpression expression) {
return expression;
}
@Override
public Expression visit(final Parentheses parentheses) {
return new Parentheses(parentheses.getExpression().visit(this));
}
@Override
public Expression visit(final Parentheses parentheses) {
return new Parentheses(parentheses.getExpression().visit(this));
}
@Override
public Expression visit(final AndCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndNotCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndNotCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final CaretAndExpression expression) {
return expression;
}
@Override
public Expression visit(final CaretAndExpression expression) {
return expression;
}
}

316
data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java
View File

@@ -21,203 +21,203 @@ import org.slf4j.LoggerFactory;
public class NewProposerParser implements QueryConstants {
private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class);
private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class);
private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose");
private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose");
/*
* Regex matching a java identifier without a caret marker. We define it as a
* blacklist, because this is easer. The regex is only used <em>after</em> the
* query has already been validated with the proper grammar.
*/
private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*";
/*
* Regex matching a java identifier without a caret marker. We define it as a
* blacklist, because this is easer. The regex is only used <em>after</em> the
* query has already been validated with the proper grammar.
*/
private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*";
private final QueryCompletionIndex queryCompletionIndex;
private final QueryCompletionIndex queryCompletionIndex;
public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) {
this.queryCompletionIndex = queryCompletionIndex;
}
public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) {
this.queryCompletionIndex = queryCompletionIndex;
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final long start = System.nanoTime();
List<Proposal> proposals;
if (StringUtils.isBlank(query.getQuery())) {
proposals = proposeForAllKeys(query.getDateRange());
} else {
public List<Proposal> propose(final QueryWithCaretMarker query) {
final long start = System.nanoTime();
List<Proposal> proposals;
if (StringUtils.isBlank(query.getQuery())) {
proposals = proposeForAllKeys(query.getDateRange());
} else {
final List<Proposal> foundProposals = proposalsForValues(query);
if (foundProposals.isEmpty()) {
proposals = proposalsForNonValues(query);
} else {
proposals = foundProposals;
}
}
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
final List<Proposal> foundProposals = proposalsForValues(query);
if (foundProposals.isEmpty()) {
proposals = proposalsForNonValues(query);
} else {
proposals = foundProposals;
}
}
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query);
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query);
return nonEmptyProposals;
}
return nonEmptyProposals;
}
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
/*
* This method is called when the query could not be parsed. It is likely that
* the next word is either a field or an operator. But is is also possible that
* the next word is a field-value, because the syntax error might be at another
* location in the query (not at the caret position).
*/
/*
* This method is called when the query could not be parsed. It is likely that
* the next word is either a field or an operator. But is is also possible that
* the next word is a field-value, because the syntax error might be at another
* location in the query (not at the caret position).
*/
final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker());
final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER));
final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker());
final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER));
if (indexTokenWithCaret > 0) {
final String previousToken = tokens.get(indexTokenWithCaret - 1);
switch (previousToken) {
case "(":
case "and":
case "or":
case "!":
proposals.addAll(proposeForAllKeys(query));
break;
if (indexTokenWithCaret > 0) {
final String previousToken = tokens.get(indexTokenWithCaret - 1);
switch (previousToken) {
case "(":
case "and":
case "or":
case "!":
proposals.addAll(proposeForAllKeys(query));
break;
case ")":
default:
// proposals.addAll(proposal);
break;
}
} else if (indexTokenWithCaret == 0) {
proposals.addAll(proposeForAllKeys(query));
}
case ")":
default:
// proposals.addAll(proposal);
break;
}
} else if (indexTokenWithCaret == 0) {
proposals.addAll(proposeForAllKeys(query));
}
return proposals;
}
return proposals;
}
private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker());
private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker());
if (wordPrefix != null) {
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange());
for (final String field : allFields) {
if (wordPrefix != null) {
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange());
for (final String field : allFields) {
if (!field.startsWith(wordPrefix)) {
continue;
}
if (!field.startsWith(wordPrefix)) {
continue;
}
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
final String proposedQuery = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* ");
final String newQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER);
final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
}
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
final String proposedQuery = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* ");
final String newQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER);
final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
}
return proposals;
}
return proposals;
}
private String wordPrefix(final String queryWithCaretMarker) {
private String wordPrefix(final String queryWithCaretMarker) {
final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")");
final Matcher matcher = pattern.matcher(queryWithCaretMarker);
if (matcher.find()) {
final String group = matcher.group();
return group.replace(CARET_MARKER, "");
}
final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")");
final Matcher matcher = pattern.matcher(queryWithCaretMarker);
if (matcher.find()) {
final String group = matcher.group();
return group.replace(CARET_MARKER, "");
}
return null;
}
return null;
}
private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) {
final List<Proposal> proposals = new ArrayList<>();
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange);
for (final String field : allFields) {
final String proposedQuery = field + "=*";
final String newQuery = field + "=";
final int newCaretPosition = newQuery.length();
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange);
for (final String field : allFields) {
final String proposedQuery = field + "=*";
final String newQuery = field + "=";
final int newCaretPosition = newQuery.length();
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
return proposals;
}
return proposals;
}
List<Proposal> proposalsForValues(final QueryWithCaretMarker query) {
try {
// Add caret marker, so that we know where the caret is.
// This also makes sure that a query like "name=|" ('|' is the caret) can be
// parsed.
// Without the caret marker the query would be "name=", which is not a valid
// expression.
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
List<Proposal> proposalsForValues(final QueryWithCaretMarker query) {
try {
// Add caret marker, so that we know where the caret is.
// This also makes sure that a query like "name=|" ('|' is the caret) can be
// parsed.
// Without the caret marker the query would be "name=", which is not a valid
// expression.
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
// parse the query
final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker);
// parse the query
final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker);
// normalize it, so that we can use the queryCompletionIndex to search for
// candidate values
final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer();
final Expression normalizedExpression = optimizer.normalizeExpression(expression);
// normalize it, so that we can use the queryCompletionIndex to search for
// candidate values
final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer();
final Expression normalizedExpression = optimizer.normalizeExpression(expression);
// find all candidate values
final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
// find all candidate values
final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker);
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker);
// translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
// translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
return proposals;
} catch (final SyntaxException e) {
LOGGER.debug("Query ({}) is not valid. This is expected to happen "
+ "unless we are looking for proposals of values.", query, e);
return Collections.emptyList();
}
}
return proposals;
} catch (final SyntaxException e) {
LOGGER.debug("Query ({}) is not valid. This is expected to happen "
+ "unless we are looking for proposals of values.", query, e);
return Collections.emptyList();
}
}
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) {
switch (resultMode) {
case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES:
return candidateValues;
default:
throw new IllegalArgumentException("Unexpected value: " + resultMode);
}
}
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) {
switch (resultMode) {
case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES:
return candidateValues;
default:
throw new IllegalArgumentException("Unexpected value: " + resultMode);
}
}
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker);
}
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker);
}
private List<Proposal> generateProposals(final String queryWithCaretMarker,
final SortedSet<String> candidateValues) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> generateProposals(final String queryWithCaretMarker,
final SortedSet<String> candidateValues) {
final List<Proposal> proposals = new ArrayList<>();
for (final String proposedTag : candidateValues) {
for (final String proposedTag : candidateValues) {
final String proposedQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER);
final String proposedQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER);
final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER);
final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition);
proposals.add(proposal);
}
final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition);
proposals.add(proposal);
}
return proposals;
}
return proposals;
}
}

368
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionExpressionOptimizer.java
View File

@@ -45,228 +45,228 @@ import org.slf4j.LoggerFactory;
*/
public class QueryCompletionExpressionOptimizer {
private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class);
private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class);
private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor {
private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor {
@Override
public Expression visit(final InExpression expression) {
if (expression.containsCaret() || expression.getValues().size() == 1) {
final String property = expression.getProperty();
final List<String> values = expression.getValues();
@Override
public Expression visit(final InExpression expression) {
if (expression.containsCaret() || expression.getValues().size() == 1) {
final String property = expression.getProperty();
final List<String> values = expression.getValues();
final List<Property> propertyExpressions = new ArrayList<>();
final List<Property> propertyExpressions = new ArrayList<>();
for (final String value : values) {
propertyExpressions.add(new Property(property, new Terminal(value)));
}
for (final String value : values) {
propertyExpressions.add(new Property(property, new Terminal(value)));
}
return Expression.Or.create(propertyExpressions);
} else {
return super.visit(expression);
}
};
}
return Expression.Or.create(propertyExpressions);
} else {
return super.visit(expression);
}
};
}
private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left.containsCaret() && !right.containsCaret()) {
return left;
}
if (!left.containsCaret() && right.containsCaret()) {
return right;
}
return super.visit(expression);
};
}
if (left.containsCaret() && !right.containsCaret()) {
return left;
}
if (!left.containsCaret() && right.containsCaret()) {
return right;
}
return super.visit(expression);
};
}
private static final class DistributiveNormalization extends IdentityExpressionVisitor {
private static final class DistributiveNormalization extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left instanceof Or) {
// (a or b) and c
// becomes
// a and c or b and c
final Expression ac = new And(((Or) left).getLeft(), right);
final Expression bc = new And(((Or) left).getRight(), right);
return new Or(ac, bc);
}
if (left instanceof Or) {
// (a or b) and c
// becomes
// a and c or b and c
final Expression ac = new And(((Or) left).getLeft(), right);
final Expression bc = new And(((Or) left).getRight(), right);
return new Or(ac, bc);
}
if (right instanceof Or) {
// a and (b or c)
// becomes
// a and b or a and c
final Expression ab = new And(left, ((Or) right).getLeft());
final Expression ac = new And(left, ((Or) right).getRight());
return new Or(ab, ac);
}
return super.visit(expression);
};
}
if (right instanceof Or) {
// a and (b or c)
// becomes
// a and b or a and c
final Expression ab = new And(left, ((Or) right).getLeft());
final Expression ac = new And(left, ((Or) right).getRight());
return new Or(ab, ac);
}
return super.visit(expression);
};
}
private static final class RotateAndExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
private static final class RotateAndExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
// (| and a) and b => | and (a and b)
//
// The expression with the caret is moved up
if (left.containsCaret() && left instanceof And) {
final Expression leftLeft = ((And) left).getLeft();
final Expression leftRight = ((And) left).getRight();
// (| and a) and b => | and (a and b)
//
// The expression with the caret is moved up
if (left.containsCaret() && left instanceof And) {
final Expression leftLeft = ((And) left).getLeft();
final Expression leftRight = ((And) left).getRight();
if (leftLeft.containsCaret()) {
return new And(leftLeft, new And(leftRight, right));
} else {
return new And(new And(leftLeft, right), leftRight);
}
} else if (right.containsCaret() && right instanceof And) {
final Expression rightLeft = ((And) right).getLeft();
final Expression rightRight = ((And) right).getRight();
if (leftLeft.containsCaret()) {
return new And(leftLeft, new And(leftRight, right));
} else {
return new And(new And(leftLeft, right), leftRight);
}
} else if (right.containsCaret() && right instanceof And) {
final Expression rightLeft = ((And) right).getLeft();
final Expression rightRight = ((And) right).getRight();
if (rightLeft.containsCaret()) {
return new And(rightLeft, new And(rightRight, left));
} else {
return new And(new And(rightLeft, left), rightRight);
}
}
if (rightLeft.containsCaret()) {
return new And(rightLeft, new And(rightRight, left));
} else {
return new And(new And(rightLeft, left), rightRight);
}
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
private static final class DoubleNegationExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression instanceof Not) {
if (expression.getExpression() instanceof Not) {
return ((Not) expression.getExpression()).getExpression();
}
}
return super.visit(expression);
}
}
private static final class DoubleNegationExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression instanceof Not) {
if (expression.getExpression() instanceof Not) {
return ((Not) expression.getExpression()).getExpression();
}
}
return super.visit(expression);
}
}
private static final class DeMorgan extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
private static final class DeMorgan extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression.getExpression() instanceof And) {
final And andExpression = (And) expression.getExpression();
final Expression left = andExpression.getLeft();
final Expression right = andExpression.getRight();
if (expression.getExpression() instanceof And) {
final And andExpression = (And) expression.getExpression();
final Expression left = andExpression.getLeft();
final Expression right = andExpression.getRight();
final Expression notLeft = new Not(left);
final Expression notRight = new Not(right);
final Expression notLeft = new Not(left);
final Expression notRight = new Not(right);
return new Or(notLeft, notRight);
}
return new Or(notLeft, notRight);
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
private static final class ToAndCaretExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
private static final class ToAndCaretExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left.containsCaret() && left instanceof Property) {
return new AndCaretExpression((Property) left, right);
}
if (right.containsCaret() && right instanceof Property) {
return new AndCaretExpression((Property) right, left);
}
if (left.containsCaret() && left instanceof Property) {
return new AndCaretExpression((Property) left, right);
}
if (right.containsCaret() && right instanceof Property) {
return new AndCaretExpression((Property) right, left);
}
if (left.containsCaret()//
&& left instanceof Not//
&& ((Not) left).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right);
}
if (right.containsCaret()//
&& right instanceof Not//
&& ((Not) right).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left);
}
if (left.containsCaret()//
&& left instanceof Not//
&& ((Not) left).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right);
}
if (right.containsCaret()//
&& right instanceof Not//
&& ((Not) right).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left);
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
public Expression normalizeExpression(final Expression expression) {
public Expression normalizeExpression(final Expression expression) {
Expression normalizingExpression = expression;
Expression previousExpression = normalizingExpression;
do {
previousExpression = normalizingExpression;
// replace all IN-expression, because they are just syntactic sugar for
// OR-expressions, but only for those that include the caret
normalizingExpression = normalizingExpression
.visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor());
Expression normalizingExpression = expression;
Expression previousExpression = normalizingExpression;
do {
previousExpression = normalizingExpression;
// replace all IN-expression, because they are just syntactic sugar for
// OR-expressions, but only for those that include the caret
normalizingExpression = normalizingExpression
.visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor());
// Remove expressions that are OR'ed with the one that contains the caret.
// Everything that is OR'ed with the 'caret'-expression cannot change the
// possible values.
normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions());
// Remove expressions that are OR'ed with the one that contains the caret.
// Everything that is OR'ed with the 'caret'-expression cannot change the
// possible values.
normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions());
// In the end we want to have expressions like "firstname=Jane and lastname=|".
// To reach that goal we use the distributive law to modify expressions like
// "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and
// lastname=|) or (firstname=John and lastname=|)"
normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization());
// In the end we want to have expressions like "firstname=Jane and lastname=|".
// To reach that goal we use the distributive law to modify expressions like
// "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and
// lastname=|) or (firstname=John and lastname=|)"
normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization());
// (fn=John and (fn=John and ln=|)
// normalized to
// (fn=John and ln=|) and (fn=Jane and ln=|)
// or normalized to
// (fn=John and fn=Jane) and ln=|
normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions());
// (fn=John and (fn=John and ln=|)
// normalized to
// (fn=John and ln=|) and (fn=Jane and ln=|)
// or normalized to
// (fn=John and fn=Jane) and ln=|
normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions());
// normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might
// later be removed
// not ( a and b) => (not a) or (not b)
normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan());
// normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might
// later be removed
// not ( a and b) => (not a) or (not b)
normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan());
// remove double negation
// not not a => a
normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions());
} while (!normalizingExpression.equals(previousExpression));
// remove double negation
// not not a => a
normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions());
} while (!normalizingExpression.equals(previousExpression));
// Replaces all (a and |) expressions with a special expression that represents
// it.
// This special expression will then be used during evaluation.
return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions());
}
// Replaces all (a and |) expressions with a special expression that represents
// it.
// This special expression will then be used during evaluation.
return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions());
}
private static Expression visitRepeatedly(final Expression expression,
final ExpressionVisitor<Expression> visitor) {
Expression previousExpression;
Expression result = expression;
private static Expression visitRepeatedly(final Expression expression,
final ExpressionVisitor<Expression> visitor) {
Expression previousExpression;
Expression result = expression;
do {
previousExpression = result;
result = previousExpression.visit(visitor);
if (!previousExpression.equals(result)) {
LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName());
LOGGER.debug(" in: {}", previousExpression);
LOGGER.debug(" out: {}", result);
}
} while (!previousExpression.equals(result));
do {
previousExpression = result;
result = previousExpression.visit(visitor);
if (!previousExpression.equals(result)) {
LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName());
LOGGER.debug(" in: {}", previousExpression);
LOGGER.debug(" out: {}", result);
}
} while (!previousExpression.equals(result));
return result;
}
return result;
}
}

184
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguage.java
View File

@@ -28,125 +28,125 @@ import org.lucares.utils.CollectionUtils;
public class QueryLanguage {
public Expression parse(final String input) {
// define the input
final CharStream in = CharStreams.fromString(input);
public Expression parse(final String input) {
// define the input
final CharStream in = CharStreams.fromString(input);
// create lexer and parser
final PdbLangLexer lexer = new PdbLangLexer(in);
lexer.addErrorListener(new ErrorListener());
// create lexer and parser
final PdbLangLexer lexer = new PdbLangLexer(in);
lexer.addErrorListener(new ErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final PdbLangParser parser = new PdbLangParser(tokens);
parser.addErrorListener(new ErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final PdbLangParser parser = new PdbLangParser(tokens);
parser.addErrorListener(new ErrorListener());
final Stack<Expression> stack = new Stack<>();
final Stack<Expression> stack = new Stack<>();
// define a listener that is called for every terminals and
// non-terminals
final ParseTreeListener listener = new PdbLangBaseListener() {
// define a listener that is called for every terminals and
// non-terminals
final ParseTreeListener listener = new PdbLangBaseListener() {
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
stack.push(new Terminal(ctx.getText()));
}
stack.push(new Terminal(ctx.getText()));
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
stack.push(new Terminal(ctx.getText()));
}
stack.push(new Terminal(ctx.getText()));
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
final Expression expression = stack.pop();
final Expression expression = stack.pop();
final Expression notExpression = new Not(expression);
stack.push(notExpression);
}
final Expression notExpression = new Not(expression);
stack.push(notExpression);
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new AndTemporary();
final Expression left = stack.pop();
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new AndTemporary();
final Expression left = stack.pop();
stack.push(operation.toExpression(left, right));
}
stack.push(operation.toExpression(left, right));
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new OrTemporary();
final Expression left = stack.pop();
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new OrTemporary();
final Expression left = stack.pop();
stack.push(operation.toExpression(left, right));
}
stack.push(operation.toExpression(left, right));
}
@Override
public void exitListOfPropValues(final ListOfPropValuesContext ctx) {
final Expression topStackElement = stack.pop();
@Override
public void exitListOfPropValues(final ListOfPropValuesContext ctx) {
final Expression topStackElement = stack.pop();
if (topStackElement instanceof ListOfPropertyValues) {
// there are at least two property values in the query
// e.g. in the expression "bird in (eagle, pigeon)"
final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement;
final Terminal nextPropertyValue = (Terminal) stack.pop();
if (topStackElement instanceof ListOfPropertyValues) {
// there are at least two property values in the query
// e.g. in the expression "bird in (eagle, pigeon)"
final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement;
final Terminal nextPropertyValue = (Terminal) stack.pop();
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue,
existingList);
stack.push(newListOfPropertyValues);
} else {
// this is the first or the only value in this list of property values
// e.g. in the expression "bird in (eagle)"
final Terminal propertyValue = (Terminal) topStackElement;
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue,
existingList);
stack.push(newListOfPropertyValues);
} else {
// this is the first or the only value in this list of property values
// e.g. in the expression "bird in (eagle)"
final Terminal propertyValue = (Terminal) topStackElement;
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue);
stack.push(newListOfPropertyValues);
}
}
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue);
stack.push(newListOfPropertyValues);
}
}
@Override
public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) {
@Override
public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) {
final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop();
final Terminal propertyName = (Terminal) stack.pop();
final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop();
final Terminal propertyName = (Terminal) stack.pop();
final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues());
stack.push(inExpression);
}
};
final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues());
stack.push(inExpression);
}
};
// Specify our entry point
final ParseTree parseTree = parser.start();
// Specify our entry point
final ParseTree parseTree = parser.start();
// Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree);
// Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree);
if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack);
}
if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack);
}
return stack.pop();
}
return stack.pop();
}
public static List<String> getTokens(final String input) {
final CharStream in = CharStreams.fromString(input);
public static List<String> getTokens(final String input) {
final CharStream in = CharStreams.fromString(input);
final PdbLangLexer lexer = new PdbLangLexer(in);
final PdbLangLexer lexer = new PdbLangLexer(in);
final CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
final List<Token> tokenList = tokens.getTokens();
return CollectionUtils.map(tokenList, Token::getText);
}
final CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
final List<Token> tokenList = tokens.getTokens();
return CollectionUtils.map(tokenList, Token::getText);
}
}

20
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguageParser.java
View File

@@ -3,15 +3,15 @@ package org.lucares.pdb.datastore.lang;
import org.apache.commons.lang3.StringUtils;
public class QueryLanguageParser {
public static Expression parse(final String query) {
public static Expression parse(final String query) {
final Expression result;
if (StringUtils.isEmpty(query)) {
result = Expression.matchAll();
} else {
final QueryLanguage lang = new QueryLanguage();
result = lang.parse(query);
}
return result;
}
final Expression result;
if (StringUtils.isEmpty(query)) {
result = Expression.matchAll();
} else {
final QueryLanguage lang = new QueryLanguage();
result = lang.parse(query);
}
return result;
}
}

90
data-store/src/main/java/org/lucares/pdb/datastore/lang/SyntaxException.java
View File

@@ -4,61 +4,61 @@ import org.antlr.v4.runtime.ParserRuleContext;
public class SyntaxException extends RuntimeException {
private static final long serialVersionUID = 1L;
private int lineStart;
private int startIndex;
private int lineStop;
private int stopIndex;
private static final long serialVersionUID = 1L;
private int lineStart;
private int startIndex;
private int lineStop;
private int stopIndex;
public SyntaxException(final ParserRuleContext context, final String message) {
this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(),
context.getStop().getStopIndex());
}
public SyntaxException(final ParserRuleContext context, final String message) {
this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(),
context.getStop().getStopIndex());
}
public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex));
this.lineStart = lineStart;
this.startIndex = startIndex;
this.lineStop = lineStop;
this.stopIndex = stopIndex;
}
public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex));
this.lineStart = lineStart;
this.startIndex = startIndex;
this.lineStop = lineStop;
this.stopIndex = stopIndex;
}
private static String generateMessage(final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
private static String generateMessage(final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
}
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
}
public int getLineStart() {
return lineStart;
}
public int getLineStart() {
return lineStart;
}
public void setLineStart(final int lineStart) {
this.lineStart = lineStart;
}
public void setLineStart(final int lineStart) {
this.lineStart = lineStart;
}
public int getStartIndex() {
return startIndex;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(final int startIndex) {
this.startIndex = startIndex;
}
public void setStartIndex(final int startIndex) {
this.startIndex = startIndex;
}
public int getLineStop() {
return lineStop;
}
public int getLineStop() {
return lineStop;
}
public void setLineStop(final int lineStop) {
this.lineStop = lineStop;
}
public void setLineStop(final int lineStop) {
this.lineStop = lineStop;
}
public int getStopIndex() {
return stopIndex;
}
public int getStopIndex() {
return stopIndex;
}
public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex;
}
public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex;
}
}

460
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -42,293 +42,293 @@ import org.testng.annotations.Test;
@Test
public class DataStoreTest {
private Path dataDirectory;
private DataStore dataStore;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
dataStore = null;
tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null;
}
public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory);
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim));
assertSearch(dateRange, "bird=eagle", eagleTim);
assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name=Tim", eagleTim, labradorTim);
assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim);
assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny);
assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny);
private Path dataDirectory;
private DataStore dataStore;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
dataStore = null;
tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null;
}
public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory);
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim));
assertSearch(dateRange, "bird=eagle", eagleTim);
assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name=Tim", eagleTim, labradorTim);
assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim);
assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny);
assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny);
// a͟n͟d binds stronger than o͟r
assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim);
assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim);
// a͟n͟d binds stronger than o͟r
assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim);
assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim);
// parenthesis override priority of a͟n͟d
assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim);
assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim);
// parenthesis override priority of a͟n͟d
assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim);
assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim);
// wildcards
assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim);
// wildcards
assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim);
// 'in' queries
assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim);
assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim);
// 'in' queries
assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim);
assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim);
}
}
public void testGetByTags() throws IOException {
public void testGetByTags() throws IOException {
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
}
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
}
public void testBlockAlignment() throws IOException {
public void testBlockAlignment() throws IOException {
dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0);
}
dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0);
}
@DataProvider(name = "providerProposals")
public Iterator<Object[]> providerProposals() {
@DataProvider(name = "providerProposals")
public Iterator<Object[]> providerProposals() {
final List<Object[]> result = new ArrayList<>();
final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") });
result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") });
// returns Tim, because it is the only dog's name starting with 'Ti'
result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") });
// returns Tim, because it is the only dog's name starting with 'Ti'
result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") });
// all cats
result.add(new Object[] { "type=cat and !name=|", "name",
Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") });
// all cats
result.add(new Object[] { "type=cat and !name=|", "name",
Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") });
// finds nothing, because there are not dogs names neither Jenny, nor Ti*
result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() });
// finds nothing, because there are not dogs names neither Jenny, nor Ti*
result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() });
result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name",
Arrays.asList("Jenny", "Tim") });
result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name",
Arrays.asList("Jenny", "Tim") });
// all but Jennifer
result.add(new Object[] { "!(type=bird) and name=|", "name",
Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") });
// all but Jennifer
result.add(new Object[] { "!(type=bird) and name=|", "name",
Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") });
result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") });
result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") });
// DeMorgan
// TODO should only match "Jenny", because Jenny is the only non-bird name
// starting with 'Jen'
result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") });
// DeMorgan
// TODO should only match "Jenny", because Jenny is the only non-bird name
// starting with 'Jen'
result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") });
result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name",
Arrays.asList("Jennifer", "Jenny", "Tim") });
result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name",
Arrays.asList("Jennifer", "Jenny", "Tim") });
// not existing field
result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() });
// not existing field
result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() });
// not existing value
result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() });
// not existing value
result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() });
return result.iterator();
}
return result.iterator();
}
@Test(dataProvider = "providerProposals")
public void testProposals(final String queryWithCaret, final String field,
final List<String> expectedProposedValues) throws Exception {
@Test(dataProvider = "providerProposals")
public void testProposals(final String queryWithCaret, final String field,
final List<String> expectedProposedValues) throws Exception {
dataStore = new DataStore(dataDirectory);
final ParititionId partitionId = DateIndexExtension.now();
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
dataStore = new DataStore(dataDirectory);
final ParititionId partitionId = DateIndexExtension.now();
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
assertProposals(dateRange, queryWithCaret, field, expectedProposedValues);
}
assertProposals(dateRange, queryWithCaret, field, expectedProposedValues);
}
public void testIdenticalDatesGoIntoSameFile() throws Exception {
public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (final DataStore dataStore = new DataStore(dataDirectory)) {
try (final DataStore dataStore = new DataStore(dataDirectory)) {
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2);
dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2);
Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache");
}
}
Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache");
}
}
public static void main(final String[] args) throws IOException, InterruptedException {
final Path dir = Files.createTempDirectory("pdb");
try (final DataStore dataStore = new DataStore(dir)) {
public static void main(final String[] args) throws IOException, InterruptedException {
final Path dir = Files.createTempDirectory("pdb");
try (final DataStore dataStore = new DataStore(dir)) {
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"),
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
final JFrame frame = new JFrame();
final JTextField input = new JTextField();
final JTextArea output = new JTextArea();
final JTextArea info = new JTextArea();
final JFrame frame = new JFrame();
final JTextField input = new JTextField();
final JTextArea output = new JTextArea();
final JTextArea info = new JTextArea();
frame.add(input, BorderLayout.NORTH);
frame.add(output, BorderLayout.CENTER);
frame.add(info, BorderLayout.SOUTH);
frame.add(input, BorderLayout.NORTH);
frame.add(output, BorderLayout.CENTER);
frame.add(info, BorderLayout.SOUTH);
input.setText("type=bird and !subtype=eagle and name=");
input.setText("type=bird and !subtype=eagle and name=");
input.addKeyListener(new KeyAdapter() {
input.addKeyListener(new KeyAdapter() {
@Override
public void keyReleased(final KeyEvent e) {
@Override
public void keyReleased(final KeyEvent e) {
final String query = input.getText();
final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT);
final String query = input.getText();
final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT);
final List<Proposal> proposals = dataStore.propose(q);
final List<Proposal> proposals = dataStore.propose(q);
final StringBuilder out = new StringBuilder();
final StringBuilder out = new StringBuilder();
for (final Proposal proposal : proposals) {
out.append(proposal.getProposedTag());
out.append(" ");
out.append(proposal.getProposedQuery());
out.append("\n");
}
for (final Proposal proposal : proposals) {
out.append(proposal.getProposedTag());
out.append(" ");
out.append(proposal.getProposedQuery());
out.append("\n");
}
final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString();
final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString();
out.append("\n");
out.append("\n");
out.append("input: " + queryWithCaretMarker);
out.append("\n");
out.append("\n");
out.append("input: " + queryWithCaretMarker);
output.setText(out.toString());
output.setText(out.toString());
}
});
final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS)));
final StringBuilder out = new StringBuilder();
out.append("info\n");
for (final Doc doc : docs) {
out.append(doc.getTags());
out.append("\n");
}
info.setText(out.toString());
}
});
final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS)));
final StringBuilder out = new StringBuilder();
out.append("info\n");
for (final Doc doc : docs) {
out.append(doc.getTags());
out.append("\n");
}
info.setText(out.toString());
frame.setSize(800, 600);
frame.setVisible(true);
TimeUnit.HOURS.sleep(1000);
}
}
frame.setSize(800, 600);
frame.setVisible(true);
TimeUnit.HOURS.sleep(1000);
}
}
private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field,
final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));
proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery()));
final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag);
Collections.sort(proposedValues);
Collections.sort(expectedProposedValues);
Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:");
}
private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) {
final List<Doc> result = dataStore.search(new Query(query, dateRange));
Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't.");
}
private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) {
final Query query = new Query(queryString, dateRange);
final List<Doc> actualDocs = dataStore.search(query);
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual);
}
private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field,
final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));
proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery()));
final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag);
Collections.sort(proposedValues);
Collections.sort(expectedProposedValues);
Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:");
}
private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) {
final List<Doc> result = dataStore.search(new Query(query, dateRange));
Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't.");
}
private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) {
final Query query = new Query(queryString, dateRange);
final List<Doc> actualDocs = dataStore.search(query);
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual);
}
}

198
data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java
View File

@@ -16,129 +16,129 @@ import org.testng.annotations.Test;
@Test
public class DateIndexExtensionTest {
@DataProvider
public Object[][] provider() {
@DataProvider
public Object[][] provider() {
final List<Object[]> result = new ArrayList<>();
final List<Object[]> result = new ArrayList<>();
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected });
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected });
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected });
}
return result.toArray(new Object[0][]);
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
@Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
final DateTimeRange dateRange = new DateTimeRange(start, end);
final DateTimeRange dateRange = new DateTimeRange(start, end);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
Assert.assertEquals(actual, expected);
}
Assert.assertEquals(actual, expected);
}
public void testDateToDateIndexPrefix() {
public void testDateToDateIndexPrefix() {
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
}
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
}
public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12));
public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12));
final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802);
final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801);
final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712);
final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802);
final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801);
final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712);
final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension
.toPartitionIds(range_201712_201802);
Assert.assertEquals(dateIndexPrefixesWithEmptyCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")));
final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension
.toPartitionIds(range_201712_201802);
Assert.assertEquals(dateIndexPrefixesWithEmptyCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")));
final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension
.toPartitionIds(range_201712_201801);
Assert.assertEquals(dateIndexPrefixesWithFilledCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801")));
final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension
.toPartitionIds(range_201712_201801);
Assert.assertEquals(dateIndexPrefixesWithFilledCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801")));
final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712);
Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712")));
}
final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712);
Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712")));
}
public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final List<Long> dateIndexEpochMillis = DateIndexExtension
.toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802));
Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802));
}
final List<Long> dateIndexEpochMillis = DateIndexExtension
.toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802));
Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802));
}
public void testPerformance() {
public void testPerformance() {
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final int iterations = 1_000_000;
final int factor = 1;
final int warmup = 20 * factor;
final int rounds = warmup + 20;
final int iterations = 1_000_000;
final int factor = 1;
final int warmup = 20 * factor;
final int rounds = warmup + 20;
// fill the cache
DateIndexExtension.DATE_PREFIX_CACHE.clear();
for (long i = min; i < max; i += 3600 * 24 * 28) {
DateIndexExtension.toPartitionId(i);
}
// fill the cache
DateIndexExtension.DATE_PREFIX_CACHE.clear();
for (long i = min; i < max; i += 3600 * 24 * 28) {
DateIndexExtension.toPartitionId(i);
}
final List<Double> measurements = new ArrayList<>();
final List<Double> measurements = new ArrayList<>();
for (int r = 0; r < rounds; r++) {
for (int r = 0; r < rounds; r++) {
final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) {
DateIndexExtension.toPartitionId(mid);
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms");
measurements.add(duration);
}
final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) {
DateIndexExtension.toPartitionId(mid);
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms");
measurements.add(duration);
}
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics();
System.out.println(stats);
}
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics();
System.out.println(stats);
}
}

465
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -22,275 +22,276 @@ import org.testng.annotations.Test;
@Test
public class ProposerTest {
private Path dataDirectory;
private DataStore dataStore;
private DateTimeRange dateRange;
private Path dataDirectory;
private DataStore dataStore;
private DateTimeRange dateRange;
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
dataStore.close();
dataStore = null;
Tags.STRING_COMPRESSOR = null;
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
dataStore.close();
dataStore = null;
Tags.STRING_COMPRESSOR = null;
}
private void initDatabase() throws Exception {
dataStore = new DataStore(dataDirectory);
dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
private void initDatabase() throws Exception {
dataStore = new DataStore(dataDirectory);
dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD);
}
dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD);
}
public void testEmptyQuery() throws Exception {
public void testEmptyQuery() throws Exception {
assertProposals("|", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
assertProposals("|", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
assertProposals(" |", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
}
assertProposals(" |", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
);
assertProposals("|bird", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
);
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim|", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("|bird", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
assertProposals("name =Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
// TODO this case is currently handled completely wrong - it is handled similar to an empty query
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim|", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("name =Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
// TODO this case is currently handled completely wrong - it is handled similar
// to an empty query
// assertProposals("|bird=eagle and name=Tim", ResultMode.FULL_VALUES, //
// new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
// new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
// );
/*
*/
}
@Test(enabled = true)
public void testInExpressions() throws Exception {
assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
/*
*/
}
assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
@Test(enabled = true)
public void testInExpressions() throws Exception {
assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
/*
*/
}
assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
public void testProposalOnEmptyValuePrefix() throws Exception {
assertProposals("name=|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
/*
*/
}
assertProposals("method=|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
);
assertProposals("method=|", ResultMode.FULL_VALUES, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
public void testProposalOnEmptyValuePrefix() throws Exception {
assertProposals("name=|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
public void testProposalOnValueSmartExpression() throws Exception {
assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
);
assertProposals("method=|", ResultMode.FULL_VALUES, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
public void testProposalOnValueSmartExpression() throws Exception {
assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
// returns nothing, because GloblikePattern.globlikeToRegex() returns the
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
// Maybe I will change that some day and allow upper case characters before
// "Stuff".
assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT);
assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34) //
);
}
// returns nothing, because GloblikePattern.globlikeToRegex() returns the
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
// Maybe I will change that some day and allow upper case characters before
// "Stuff".
assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT);
public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and |", ResultMode.FULL_VALUES, //
proposal("name", "name=* and name=* ", "name=* and name=|"), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), //
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34) //
);
}
public void testProposalWithWildcards() throws Exception {
public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and |", ResultMode.FULL_VALUES, //
proposal("name", "name=* and name=* ", "name=* and name=|"), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), //
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
assertProposals("name=*im|", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim", "name=Tim|"), //
proposal("Timothy", "name=Timothy", "name=Timothy|")//
);
public void testProposalWithWildcards() throws Exception {
assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, //
proposal("FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), //
proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
assertProposals("name=*im|", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim", "name=Tim|"), //
proposal("Timothy", "name=Timothy", "name=Timothy|")//
);
public void testProposalWithAndExpression() throws Exception {
assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), //
proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")//
);
assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, //
proposal("FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), //
proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), //
proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")//
);
}
public void testProposalWithAndExpression() throws Exception {
assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), //
proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")//
);
public void testProposalWithAndNotExpression() throws Exception {
assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle",
"name=Tim and ! dog=labrador and bird=eagle|") //
);
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), //
proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")//
);
}
private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) {
final String newQueryWithoutCaretMarker = newQuery.replace("|", "");
final int newCaretPosition = newQuery.indexOf('|');
return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition);
}
public void testProposalWithAndNotExpression() throws Exception {
assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle",
"name=Tim and ! dog=labrador and bird=eagle|") //
);
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected)
throws InterruptedException {
final int caretIndex = query.indexOf("|");
final String q = query.replace("|", "");
assertProposals(q, resultMode, caretIndex, expected);
}
private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) {
final String newQueryWithoutCaretMarker = newQuery.replace("|", "");
final int newCaretPosition = newQuery.indexOf('|');
return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition);
}
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
final Proposal... expected) throws InterruptedException {
private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected)
throws InterruptedException {
final int caretIndex = query.indexOf("|");
final String q = query.replace("|", "");
assertProposals(q, resultMode, caretIndex, expected);
}
final List<Proposal> actual = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
final Proposal... expected) throws InterruptedException {
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString)));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString)));
Assert.assertEquals(actual, expectedList);
}
final List<Proposal> actual = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString)));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString)));
Assert.assertEquals(actual, expectedList);
}
}

78
data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java
View File

@@ -21,53 +21,53 @@ import org.testng.annotations.Test;
@Test
public class QueryCompletionIndexTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void test() throws Exception {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
public void test() throws Exception {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
);
final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
);
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) {
for (final Tags t : tags) {
index.addTags(partitionId, t);
}
try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) {
for (final Tags t : tags) {
index.addTags(partitionId, t);
}
// all firstnames where lastname=Doe are returned sorted alphabetically.
// tags A and B match
final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"),
"firstname");
Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John"));
// all firstnames where lastname=Doe are returned sorted alphabetically.
// tags A and B match
final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"),
"firstname");
Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John"));
// no duplicates are returned:
// tags A and C match firstname=John, but both have country=Atlantis
final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"),
"country");
Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis"));
// no duplicates are returned:
// tags A and C match firstname=John, but both have country=Atlantis
final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"),
"country");
Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis"));
// findAllValuesForField sorts alphabetically
final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname");
Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames);
// findAllValuesForField sorts alphabetically
final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname");
Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames);
final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country");
Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado"));
}
}
final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country");
Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado"));
}
}
}

88
data-store/src/test/java/org/lucares/pdb/datastore/lang/CandidateGrouperTest.java
View File

@@ -12,54 +12,54 @@ import org.testng.annotations.Test;
@Test
public class CandidateGrouperTest {
@DataProvider
public Object[][] providerGroup() {
final List<Object[]> result = new ArrayList<>();
@DataProvider
public Object[][] providerGroup() {
final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = |", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = a|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.|", //
Set.of("aa.xx.", "aa.yy.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.x|", //
Set.of("aa.xx.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.xx.|", //
Set.of("aa.xx.AA.", "aa.xx.BB") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), //
"name = aa.xx.AA.|", //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") });
result.add(new Object[] { //
Set.of("XX.YY.ZZ", "XX.YY"), //
"name = XX.Y|", //
Set.of("XX.YY.", "XX.YY") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = |", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = a|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.|", //
Set.of("aa.xx.", "aa.yy.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.x|", //
Set.of("aa.xx.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.xx.|", //
Set.of("aa.xx.AA.", "aa.xx.BB") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), //
"name = aa.xx.AA.|", //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") });
result.add(new Object[] { //
Set.of("XX.YY.ZZ", "XX.YY"), //
"name = XX.Y|", //
Set.of("XX.YY.", "XX.YY") });
return result.toArray(new Object[0][]);
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "providerGroup")
public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) {
final CandidateGrouper grouper = new CandidateGrouper();
@Test(dataProvider = "providerGroup")
public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) {
final CandidateGrouper grouper = new CandidateGrouper();
final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER);
final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER);
final SortedSet<String> actual = grouper.group(values, query);
final SortedSet<String> actual = grouper.group(values, query);
Assert.assertEquals(actual, expected);
}
Assert.assertEquals(actual, expected);
}
}

76
file-utils/src/main/java/org/lucares/utils/file/FileUtils.java
View File

@@ -15,55 +15,55 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FileUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
private static final class RecursiveDeleter extends SimpleFileVisitor<Path> {
private static final class RecursiveDeleter extends SimpleFileVisitor<Path> {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
Files.delete(file);
LOGGER.trace("deleted: {}", file);
Files.delete(file);
LOGGER.trace("deleted: {}", file);
return FileVisitResult.CONTINUE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
Files.delete(dir);
LOGGER.trace("deleted: {}", dir);
Files.delete(dir);
LOGGER.trace("deleted: {}", dir);
return FileVisitResult.CONTINUE;
}
}
return FileVisitResult.CONTINUE;
}
}
public static void delete(final Path path) {
public static void delete(final Path path) {
final int maxAttempts = 10;
int attempt = 1;
final int maxAttempts = 10;
int attempt = 1;
while (attempt <= maxAttempts) {
try {
LOGGER.debug("deleting '{}' attempt {} of {}", path.toFile().getAbsolutePath(), attempt, maxAttempts);
Files.walkFileTree(path, new RecursiveDeleter());
break;
} catch (final IOException e) {
final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt
+ " of " + maxAttempts;
LOGGER.warn(msg, e);
}
attempt++;
}
}
while (attempt <= maxAttempts) {
try {
LOGGER.debug("deleting '{}' attempt {} of {}", path.toFile().getAbsolutePath(), attempt, maxAttempts);
Files.walkFileTree(path, new RecursiveDeleter());
break;
} catch (final IOException e) {
final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt
+ " of " + maxAttempts;
LOGGER.warn(msg, e);
}
attempt++;
}
}
public static List<Path> listRecursively(final Path start) throws IOException {
public static List<Path> listRecursively(final Path start) throws IOException {
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
return files.collect(Collectors.toList());
}
}
try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
return files.collect(Collectors.toList());
}
}
}

156
pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java
View File

@@ -8,106 +8,106 @@ import java.time.temporal.TemporalUnit;
public class DateTimeRange {
private static final DateTimeRange MAX = new DateTimeRange(
OffsetDateTime.of(1900, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC),
OffsetDateTime.of(2100, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC));
private static final DateTimeRange MAX = new DateTimeRange(
OffsetDateTime.of(1900, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC),
OffsetDateTime.of(2100, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC));
private final OffsetDateTime start;
private final OffsetDateTime end;
private final OffsetDateTime start;
private final OffsetDateTime end;
public DateTimeRange(final OffsetDateTime start, final OffsetDateTime end) {
this.start = start;
this.end = end;
}
public DateTimeRange(final OffsetDateTime start, final OffsetDateTime end) {
this.start = start;
this.end = end;
}
public static DateTimeRange max() {
return MAX;
}
public static DateTimeRange max() {
return MAX;
}
public static DateTimeRange now() {
return relativeMillis(0);
}
public static DateTimeRange now() {
return relativeMillis(0);
}
public static DateTimeRange relative(final long amount, final TemporalUnit unit) {
final OffsetDateTime now = OffsetDateTime.now();
return new DateTimeRange(now.minus(amount, unit), now);
}
public static DateTimeRange relative(final long amount, final TemporalUnit unit) {
final OffsetDateTime now = OffsetDateTime.now();
return new DateTimeRange(now.minus(amount, unit), now);
}
public static DateTimeRange relativeMillis(final long amount) {
return relative(amount, ChronoUnit.MILLIS);
}
public static DateTimeRange relativeMillis(final long amount) {
return relative(amount, ChronoUnit.MILLIS);
}
public static DateTimeRange relativeSeconds(final long amount) {
return relative(amount, ChronoUnit.SECONDS);
}
public static DateTimeRange relativeSeconds(final long amount) {
return relative(amount, ChronoUnit.SECONDS);
}
public static DateTimeRange relativeMinutes(final long amount) {
return relative(amount, ChronoUnit.MINUTES);
}
public static DateTimeRange relativeMinutes(final long amount) {
return relative(amount, ChronoUnit.MINUTES);
}
public static DateTimeRange relativeHours(final long amount) {
return relative(amount, ChronoUnit.HOURS);
}
public static DateTimeRange relativeHours(final long amount) {
return relative(amount, ChronoUnit.HOURS);
}
public static DateTimeRange relativeDays(final long amount) {
return relative(amount, ChronoUnit.DAYS);
}
public static DateTimeRange relativeDays(final long amount) {
return relative(amount, ChronoUnit.DAYS);
}
public static DateTimeRange relativeMonths(final long amount) {
return relative(amount, ChronoUnit.MONTHS);
}
public static DateTimeRange relativeMonths(final long amount) {
return relative(amount, ChronoUnit.MONTHS);
}
public static DateTimeRange relativeYears(final long amount) {
return relative(amount, ChronoUnit.YEARS);
}
public static DateTimeRange relativeYears(final long amount) {
return relative(amount, ChronoUnit.YEARS);
}
public OffsetDateTime getStart() {
return start;
}
public OffsetDateTime getStart() {
return start;
}
public long getStartEpochMilli() {
return start.toInstant().toEpochMilli();
}
public long getStartEpochMilli() {
return start.toInstant().toEpochMilli();
}
public OffsetDateTime getEnd() {
return end;
}
public OffsetDateTime getEnd() {
return end;
}
public long getEndEpochMilli() {
return end.toInstant().toEpochMilli();
}
public long getEndEpochMilli() {
return end.toInstant().toEpochMilli();
}
@Override
public String toString() {
return start + "-" + end;
}
@Override
public String toString() {
return start + "-" + end;
}
public static DateTimeRange ofDay(final OffsetDateTime day) {
final OffsetDateTime from = day.truncatedTo(ChronoUnit.DAYS);
final OffsetDateTime to = from.plusDays(1).minusNanos(1);
public static DateTimeRange ofDay(final OffsetDateTime day) {
final OffsetDateTime from = day.truncatedTo(ChronoUnit.DAYS);
final OffsetDateTime to = from.plusDays(1).minusNanos(1);
return new DateTimeRange(from, to);
}
return new DateTimeRange(from, to);
}
public Duration duration() {
return Duration.between(start, end);
}
public Duration duration() {
return Duration.between(start, end);
}
public boolean inRange(final long epochMilli) {
final long fromEpochMilli = start.toInstant().toEpochMilli();
final long toEpochMilli = end.toInstant().toEpochMilli();
public boolean inRange(final long epochMilli) {
final long fromEpochMilli = start.toInstant().toEpochMilli();
final long toEpochMilli = end.toInstant().toEpochMilli();
return fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli;
}
return fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli;
}
public boolean inRange(final OffsetDateTime date) {
return start.compareTo(date) <= 0 && end.compareTo(date) >= 0;
}
public boolean inRange(final OffsetDateTime date) {
return start.compareTo(date) <= 0 && end.compareTo(date) >= 0;
}
public boolean intersect(final DateTimeRange timeRange) {
return inRange(timeRange.start) //
|| inRange(timeRange.end) //
|| timeRange.inRange(start)//
|| timeRange.inRange(end);
}
public boolean intersect(final DateTimeRange timeRange) {
return inRange(timeRange.start) //
|| inRange(timeRange.end) //
|| timeRange.inRange(start)//
|| timeRange.inRange(end);
}
}

50
pdb-api/src/main/java/org/lucares/pdb/api/Entries.java
View File

@@ -7,36 +7,36 @@ import java.util.Iterator;
import java.util.List;
public class Entries implements Iterable<Entry> {
/**
* A special {@link Entries} instance that can be used as poison object for
* {@link BlockingQueueIterator}.
*/
public static final Entries POISON = new Entries(0);
/**
* A special {@link Entries} instance that can be used as poison object for
* {@link BlockingQueueIterator}.
*/
public static final Entries POISON = new Entries(0);
private final List<Entry> entries;
private final List<Entry> entries;
public Entries(final int initialSize) {
entries = new ArrayList<>(initialSize);
}
public Entries(final int initialSize) {
entries = new ArrayList<>(initialSize);
}
public Entries(final Entry... entries) {
this.entries = new ArrayList<>(Arrays.asList(entries));
}
public Entries(final Entry... entries) {
this.entries = new ArrayList<>(Arrays.asList(entries));
}
public Entries(final Collection<Entry> entries) {
this.entries = new ArrayList<>(entries);
}
public Entries(final Collection<Entry> entries) {
this.entries = new ArrayList<>(entries);
}
public void add(final Entry entry) {
entries.add(entry);
}
public void add(final Entry entry) {
entries.add(entry);
}
@Override
public Iterator<Entry> iterator() {
return entries.iterator();
}
@Override
public Iterator<Entry> iterator() {
return entries.iterator();
}
public int size() {
return entries.size();
}
public int size() {
return entries.size();
}
}

102
pdb-api/src/main/java/org/lucares/pdb/api/Entry.java
View File

@@ -7,65 +7,65 @@ import java.time.format.DateTimeFormatter;
public class Entry {
private final long value;
private final long value;
private final Tags tags;
private final Tags tags;
private final long epochMilli;
private final long epochMilli;
public Entry(final long epochMilli, final long value, final Tags tags) {
this.epochMilli = epochMilli;
this.tags = tags;
this.value = value;
}
public Entry(final long epochMilli, final long value, final Tags tags) {
this.epochMilli = epochMilli;
this.tags = tags;
this.value = value;
}
public long getValue() {
return value;
}
public long getValue() {
return value;
}
public long getEpochMilli() {
return epochMilli;
}
public long getEpochMilli() {
return epochMilli;
}
public Tags getTags() {
return tags;
}
public Tags getTags() {
return tags;
}
@Override
public String toString() {
@Override
public String toString() {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags.asString() + ")";
}
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags.asString() + ")";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
result = prime * result + (int) (value ^ (value >>> 32));
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
result = prime * result + (int) (value ^ (value >>> 32));
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Entry other = (Entry) obj;
if (epochMilli != other.epochMilli)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
if (value != other.value)
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Entry other = (Entry) obj;
if (epochMilli != other.epochMilli)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
if (value != other.value)
return false;
return true;
}
}

40
pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java
View File

@@ -6,31 +6,31 @@ import org.lucares.collections.LongList;
public class GroupResult {
private final Tags groupedBy;
private final Tags groupedBy;
private final Stream<LongList> timeValueStream;
private final Stream<LongList> timeValueStream;
public GroupResult(final Stream<LongList> entries, final Tags groupedBy) {
this.timeValueStream = entries;
this.groupedBy = groupedBy;
}
public GroupResult(final Stream<LongList> entries, final Tags groupedBy) {
this.timeValueStream = entries;
this.groupedBy = groupedBy;
}
public Tags getGroupedBy() {
return groupedBy;
}
public Tags getGroupedBy() {
return groupedBy;
}
/**
* @return {@link Stream}
*/
public Stream<LongList> asStream() {
return timeValueStream;
}
/**
* @return {@link Stream}
*/
public Stream<LongList> asStream() {
return timeValueStream;
}
public LongList flatMap() {
final LongList result = new LongList();
public LongList flatMap() {
final LongList result = new LongList();
timeValueStream.forEachOrdered(result::addAll);
timeValueStream.forEachOrdered(result::addAll);
return result;
}
return result;
}
}

102
pdb-api/src/main/java/org/lucares/pdb/api/Query.java
View File

@@ -4,74 +4,74 @@ import java.util.ArrayList;
import java.util.List;
public class Query {
private final String query;
private final String query;
private final DateTimeRange dateRange;
private final DateTimeRange dateRange;
public Query(final String query, final DateTimeRange dateRange) {
super();
this.query = query;
this.dateRange = dateRange;
}
public Query(final String query, final DateTimeRange dateRange) {
super();
this.query = query;
this.dateRange = dateRange;
}
public Query relativeMillis(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMillis(amount));
}
public Query relativeMillis(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMillis(amount));
}
public Query relativeSeconds(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeSeconds(amount));
}
public Query relativeSeconds(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeSeconds(amount));
}
public Query relativeMinutes(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMinutes(amount));
}
public Query relativeMinutes(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMinutes(amount));
}
public Query relativeHours(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeHours(amount));
}
public Query relativeHours(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeHours(amount));
}
public Query relativeDays(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeDays(amount));
}
public Query relativeDays(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeDays(amount));
}
public Query relativeMonths(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMonths(amount));
}
public Query relativeMonths(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMonths(amount));
}
public static Query createQuery(final String query, final DateTimeRange dateRange) {
return new Query(query, dateRange);
}
public static Query createQuery(final String query, final DateTimeRange dateRange) {
return new Query(query, dateRange);
}
public static Query createQuery(final Tags tags, final DateTimeRange dateRange) {
public static Query createQuery(final Tags tags, final DateTimeRange dateRange) {
final List<String> terms = new ArrayList<>();
final List<String> terms = new ArrayList<>();
for (final String key : tags.getKeys()) {
final String value = tags.getValue(key);
for (final String key : tags.getKeys()) {
final String value = tags.getValue(key);
final StringBuilder term = new StringBuilder();
term.append(key);
term.append("=");
term.append(value);
term.append(" ");
final StringBuilder term = new StringBuilder();
term.append(key);
term.append("=");
term.append(value);
term.append(" ");
terms.add(term.toString());
}
terms.add(term.toString());
}
return new Query(String.join(" and ", terms), dateRange);
}
return new Query(String.join(" and ", terms), dateRange);
}
public String getQuery() {
return query;
}
public String getQuery() {
return query;
}
public DateTimeRange getDateRange() {
return dateRange;
}
public DateTimeRange getDateRange() {
return dateRange;
}
@Override
public String toString() {
return "'" + query + "' [" + dateRange + "]";
}
@Override
public String toString() {
return "'" + query + "' [" + dateRange + "]";
}
}

2
pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.api;
public interface QueryConstants {
String CARET_MARKER = "\ue001"; // character in the private use area
String CARET_MARKER = "\ue001"; // character in the private use area
}

38
pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java
View File

@@ -2,28 +2,28 @@ package org.lucares.pdb.api;
public class QueryWithCaretMarker extends Query implements QueryConstants {
public enum ResultMode {
CUT_AT_DOT, FULL_VALUES
}
public enum ResultMode {
CUT_AT_DOT, FULL_VALUES
}
private final int caretIndex;
private final ResultMode resultMode;
private final int caretIndex;
private final ResultMode resultMode;
public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex,
final ResultMode resultMode) {
super(query, dateRange);
this.caretIndex = caretIndex;
this.resultMode = resultMode;
}
public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex,
final ResultMode resultMode) {
super(query, dateRange);
this.caretIndex = caretIndex;
this.resultMode = resultMode;
}
public String getQueryWithCaretMarker() {
final StringBuilder queryBuilder = new StringBuilder(getQuery());
final StringBuilder queryWithCaretMarker = queryBuilder.insert(caretIndex, CARET_MARKER);
return queryWithCaretMarker.toString();
}
public String getQueryWithCaretMarker() {
final StringBuilder queryBuilder = new StringBuilder(getQuery());
final StringBuilder queryWithCaretMarker = queryBuilder.insert(caretIndex, CARET_MARKER);
return queryWithCaretMarker.toString();
}
public ResultMode getResultMode() {
return resultMode;
}
public ResultMode getResultMode() {
return resultMode;
}
}

32
pdb-api/src/main/java/org/lucares/pdb/api/Result.java
View File

@@ -7,24 +7,24 @@ import java.util.List;
public class Result {
private final List<GroupResult> groupResults;
private final List<GroupResult> groupResults;
public Result(final GroupResult... groupResults) {
this(Arrays.asList(groupResults));
}
public Result(final GroupResult... groupResults) {
this(Arrays.asList(groupResults));
}
public Result(final Collection<GroupResult> groupResults) {
this.groupResults = new ArrayList<>(groupResults);
}
public Result(final Collection<GroupResult> groupResults) {
this.groupResults = new ArrayList<>(groupResults);
}
public GroupResult singleGroup() {
if (groupResults.size() != 1) {
throw new IllegalStateException("the result does not contain exactly one group");
}
return groupResults.get(0);
}
public GroupResult singleGroup() {
if (groupResults.size() != 1) {
throw new IllegalStateException("the result does not contain exactly one group");
}
return groupResults.get(0);
}
public List<GroupResult> getGroups() {
return new ArrayList<>(groupResults);
}
public List<GroupResult> getGroups() {
return new ArrayList<>(groupResults);
}
}

8
pdb-api/src/main/java/org/lucares/pdb/api/RuntimeIOException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.api;
public class RuntimeIOException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public RuntimeIOException(final Throwable cause) {
super(cause);
}
public RuntimeIOException(final Throwable cause) {
super(cause);
}
}

42
pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java
View File

@@ -7,34 +7,34 @@ import java.nio.file.Path;
*/
public class StringCompressor {
private final UniqueStringIntegerPairs usip;
private final UniqueStringIntegerPairs usip;
public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException {
this.usip = usip;
}
public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException {
this.usip = usip;
}
public static StringCompressor create(final Path path) {
final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path);
return new StringCompressor(mapsi);
}
public static StringCompressor create(final Path path) {
final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path);
return new StringCompressor(mapsi);
}
public int put(final String string) {
public int put(final String string) {
return usip.computeIfAbsent(string, s -> usip.getHighestInteger() + 1);
}
return usip.computeIfAbsent(string, s -> usip.getHighestInteger() + 1);
}
public int put(final byte[] bytes, final int start, final int endExclusive) {
return usip.computeIfAbsent(bytes, start, endExclusive);
}
public int put(final byte[] bytes, final int start, final int endExclusive) {
return usip.computeIfAbsent(bytes, start, endExclusive);
}
public String get(final int integer) {
public String get(final int integer) {
return usip.getKey(integer);
}
return usip.getKey(integer);
}
public int getIfPresent(final String string) {
final Integer integer = usip.get(string);
return integer != null ? integer : -1;
}
public int getIfPresent(final String string) {
final Integer integer = usip.get(string);
return integer != null ? integer : -1;
}
}

142
pdb-api/src/main/java/org/lucares/pdb/api/Tag.java
View File

@@ -6,89 +6,89 @@ package org.lucares.pdb.api;
* 'Sam' is the value.
*/
public class Tag implements Comparable<Tag> {
private final int field;
private final int field;
private final int value;
private final int value;
/**
* Create a new tag with field and value specified as int. See
* {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints.
*
* @param field the field as int
* @param value the value as int
*/
public Tag(final int field, final int value) {
this.field = field;
this.value = value;
}
/**
* Create a new tag with field and value specified as int. See
* {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints.
*
* @param field the field as int
* @param value the value as int
*/
public Tag(final int field, final int value) {
this.field = field;
this.value = value;
}
/**
* Create a new {@link Tag} for the given field and value.
*
* @param field the field
* @param value the value
*/
public Tag(final String field, final String value) {
this.field = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1;
this.value = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1;
}
/**
* Create a new {@link Tag} for the given field and value.
*
* @param field the field
* @param value the value
*/
public Tag(final String field, final String value) {
this.field = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1;
this.value = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1;
}
@Override
public int compareTo(final Tag o) {
@Override
public int compareTo(final Tag o) {
if (field != o.field) {
return field - o.field;
} else if (value != o.value) {
return value - o.value;
}
if (field != o.field) {
return field - o.field;
} else if (value != o.value) {
return value - o.value;
}
return 0;
}
return 0;
}
public int getKey() {
return field;
}
public int getKey() {
return field;
}
public String getKeyAsString() {
return Tags.STRING_COMPRESSOR.get(field);
}
public String getKeyAsString() {
return Tags.STRING_COMPRESSOR.get(field);
}
public int getValue() {
return value;
}
public int getValue() {
return value;
}
public String getValueAsString() {
return Tags.STRING_COMPRESSOR.get(value);
}
public String getValueAsString() {
return Tags.STRING_COMPRESSOR.get(value);
}
@Override
public String toString() {
return Tags.STRING_COMPRESSOR.get(field) + "=" + Tags.STRING_COMPRESSOR.get(value);
}
@Override
public String toString() {
return Tags.STRING_COMPRESSOR.get(field) + "=" + Tags.STRING_COMPRESSOR.get(value);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + field;
result = prime * result + value;
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + field;
result = prime * result + value;
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Tag other = (Tag) obj;
if (field != other.field)
return false;
if (value != other.value)
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Tag other = (Tag) obj;
if (field != other.field)
return false;
if (value != other.value)
return false;
return true;
}
}

2
pdb-api/src/main/java/org/lucares/pdb/api/TagByKeyAndValueComparator.java
View File

@@ -4,5 +4,5 @@ import java.util.Comparator;
public class TagByKeyAndValueComparator {
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey).thenComparing(Tag::getValue);
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey).thenComparing(Tag::getValue);
}

2
pdb-api/src/main/java/org/lucares/pdb/api/TagByKeyComparator.java
View File

@@ -4,5 +4,5 @@ import java.util.Comparator;
public class TagByKeyComparator {
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey);
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey);
}

408
pdb-api/src/main/java/org/lucares/pdb/api/Tags.java
View File

@@ -14,264 +14,264 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
public class Tags implements Comparable<Tags> {
public static StringCompressor STRING_COMPRESSOR = null;
private static final byte[] EMPTY_BYTES = new byte[0];
public static final Tags EMPTY = new Tags();
public static StringCompressor STRING_COMPRESSOR = null;
private static final byte[] EMPTY_BYTES = new byte[0];
public static final Tags EMPTY = new Tags();
private final List<Tag> tags;
private final List<Tag> tags;
public Tags() {
tags = new ArrayList<>();
}
public Tags() {
tags = new ArrayList<>();
}
public Tags(final List<Tag> tags) {
Collections.sort(tags, TagByKeyAndValueComparator.INSTANCE);
this.tags = tags;
}
public Tags(final List<Tag> tags) {
Collections.sort(tags, TagByKeyAndValueComparator.INSTANCE);
this.tags = tags;
}
public static Tags create(final List<Tag> tags) {
public static Tags create(final List<Tag> tags) {
return new Tags(tags);
}
return new Tags(tags);
}
public static Tags create() {
return EMPTY;
}
public static Tags create() {
return EMPTY;
}
public static Tags create(final int key, final int value) {
public static Tags create(final int key, final int value) {
return TagsBuilder.create().add(key, value).build();
}
return TagsBuilder.create().add(key, value).build();
}
public static Tags create(final int key1, final int value1, final int key2, final int value2) {
public static Tags create(final int key1, final int value1, final int key2, final int value2) {
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).build();
return result;
}
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).build();
return result;
}
public static Tags create(final int key1, final int value1, final int key2, final int value2, final int key3,
final int value3) {
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).add(key3, value3).build();
return result;
}
public static Tags create(final int key1, final int value1, final int key2, final int value2, final int key3,
final int value3) {
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).add(key3, value3).build();
return result;
}
public static Tags createAndAddToDictionary(final String key, final String value) {
public static Tags createAndAddToDictionary(final String key, final String value) {
return TagsBuilder.create().addAndAddToDictionary(key, value).build();
}
return TagsBuilder.create().addAndAddToDictionary(key, value).build();
}
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2) {
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.build();
return result;
}
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.build();
return result;
}
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).build();
return result;
}
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).build();
return result;
}
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3, final String key4, final String value4) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).addAndAddToDictionary(key4, value4).build();
return result;
}
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3, final String key4, final String value4) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).addAndAddToDictionary(key4, value4).build();
return result;
}
public static Tags fromBytes(final byte[] bytes) {
final List<Tag> result = new ArrayList<>();
public static Tags fromBytes(final byte[] bytes) {
final List<Tag> result = new ArrayList<>();
final LongList keyValuesAsLongs = VariableByteEncoder.decode(bytes);
final LongList keyValuesAsLongs = VariableByteEncoder.decode(bytes);
for (int i = 0; i < keyValuesAsLongs.size(); i += 2) {
for (int i = 0; i < keyValuesAsLongs.size(); i += 2) {
final long keyAsLong = keyValuesAsLongs.get(i);
final long valueAsLong = keyValuesAsLongs.get(i + 1);
final long keyAsLong = keyValuesAsLongs.get(i);
final long valueAsLong = keyValuesAsLongs.get(i + 1);
final int key = (int) keyAsLong;
final int value = (int) valueAsLong;
result.add(new Tag(key, value));
}
final int key = (int) keyAsLong;
final int value = (int) valueAsLong;
result.add(new Tag(key, value));
}
return new Tags(result);
}
return new Tags(result);
}
public byte[] toBytes() {
final byte[] result;
public byte[] toBytes() {
final byte[] result;
if (tags.size() > 0) {
final LongList keyValuesAsLongs = new LongList(tags.size() * 2);
for (final Tag tag : tags) {
final long keyAsLong = tag.getKey();
final long valueAsLong = tag.getValue();
if (tags.size() > 0) {
final LongList keyValuesAsLongs = new LongList(tags.size() * 2);
for (final Tag tag : tags) {
final long keyAsLong = tag.getKey();
final long valueAsLong = tag.getValue();
keyValuesAsLongs.add(keyAsLong);
keyValuesAsLongs.add(valueAsLong);
}
keyValuesAsLongs.add(keyAsLong);
keyValuesAsLongs.add(valueAsLong);
}
result = VariableByteEncoder.encode(keyValuesAsLongs);
} else {
result = EMPTY_BYTES;
}
return result;
}
result = VariableByteEncoder.encode(keyValuesAsLongs);
} else {
result = EMPTY_BYTES;
}
return result;
}
@Override
public int compareTo(final Tags o) {
@Override
public int compareTo(final Tags o) {
if (tags.size() != o.tags.size()) {
return tags.size() - o.tags.size();
} else {
for (int i = 0; i < tags.size(); i++) {
final int compareResult = tags.get(i).compareTo(o.tags.get(i));
if (compareResult != 0) {
return compareResult;
}
}
}
if (tags.size() != o.tags.size()) {
return tags.size() - o.tags.size();
} else {
for (int i = 0; i < tags.size(); i++) {
final int compareResult = tags.get(i).compareTo(o.tags.get(i));
if (compareResult != 0) {
return compareResult;
}
}
}
return 0;
}
return 0;
}
public String getValue(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
public String getValue(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) {
final Tag tag = tags.get(index);
return STRING_COMPRESSOR.get(tag.getValue());
}
return null;
}
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) {
final Tag tag = tags.get(index);
return STRING_COMPRESSOR.get(tag.getValue());
}
return null;
}
public int getValueAsInt(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
public int getValueAsInt(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) {
final Tag tag = tags.get(index);
return tag.getValue();
}
return -1;
}
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) {
final Tag tag = tags.get(index);
return tag.getValue();
}
return -1;
}
public Set<String> getKeys() {
final TreeSet<String> result = new TreeSet<>();
for (final Tag tag : tags) {
result.add(STRING_COMPRESSOR.get(tag.getKey()));
}
return result;
}
public Set<String> getKeys() {
final TreeSet<String> result = new TreeSet<>();
for (final Tag tag : tags) {
result.add(STRING_COMPRESSOR.get(tag.getKey()));
}
return result;
}
public IntList getKeysAsInt() {
final IntList result = new IntList();
for (final Tag tag : tags) {
result.add(tag.getKey());
}
return result;
}
public IntList getKeysAsInt() {
final IntList result = new IntList();
for (final Tag tag : tags) {
result.add(tag.getKey());
}
return result;
}
public List<Tag> toTags() {
return Collections.unmodifiableList(tags);
}
public List<Tag> toTags() {
return Collections.unmodifiableList(tags);
}
public void forEach(final BiConsumer<String, String> keyValueConsumer) {
public void forEach(final BiConsumer<String, String> keyValueConsumer) {
for (final Tag tag : tags) {
final String key = STRING_COMPRESSOR.get(tag.getKey());
final String value = STRING_COMPRESSOR.get(tag.getValue());
keyValueConsumer.accept(key, value);
}
}
for (final Tag tag : tags) {
final String key = STRING_COMPRESSOR.get(tag.getKey());
final String value = STRING_COMPRESSOR.get(tag.getValue());
keyValueConsumer.accept(key, value);
}
}
public Tags mapTags(final Function<Tag, Tag> tagMapFuntion) {
final List<Tag> mappedTags = new ArrayList<>(tags.size());
for (final Tag tag : tags) {
mappedTags.add(tagMapFuntion.apply(tag));
}
return Tags.create(mappedTags);
}
public Tags mapTags(final Function<Tag, Tag> tagMapFuntion) {
final List<Tag> mappedTags = new ArrayList<>(tags.size());
for (final Tag tag : tags) {
mappedTags.add(tagMapFuntion.apply(tag));
}
return Tags.create(mappedTags);
}
@Override
public String toString() {
return String.valueOf(tags);
}
@Override
public String toString() {
return String.valueOf(tags);
}
public String toCsv() {
final List<String> tagsAsStrings = new ArrayList<>();
for (final Tag tag : tags) {
tagsAsStrings.add(tag.getKeyAsString() + "=" + tag.getValueAsString());
}
public String toCsv() {
final List<String> tagsAsStrings = new ArrayList<>();
for (final Tag tag : tags) {
tagsAsStrings.add(tag.getKeyAsString() + "=" + tag.getValueAsString());
}
return String.join(",", tagsAsStrings);
}
return String.join(",", tagsAsStrings);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Tags other = (Tags) obj;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Tags other = (Tags) obj;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
public Tags subset(final List<String> groupByFields) {
public Tags subset(final List<String> groupByFields) {
final TagsBuilder result = TagsBuilder.create();
final TagsBuilder result = TagsBuilder.create();
for (final String field : groupByFields) {
final int value = getValueAsInt(field);
for (final String field : groupByFields) {
final int value = getValueAsInt(field);
if (value >= 0) {
final int fieldAsInt = STRING_COMPRESSOR.getIfPresent(field);
result.add(fieldAsInt, value);
}
}
if (value >= 0) {
final int fieldAsInt = STRING_COMPRESSOR.getIfPresent(field);
result.add(fieldAsInt, value);
}
}
return result.build();
}
return result.build();
}
public boolean isEmpty() {
return tags.isEmpty();
}
public boolean isEmpty() {
return tags.isEmpty();
}
/**
* @return User facing readable representation
*/
public String asString() {
/**
* @return User facing readable representation
*/
public String asString() {
final StringBuilder result = new StringBuilder();
final StringBuilder result = new StringBuilder();
for (final Tag tag : tags) {
if (result.length() > 0) {
result.append(", ");
}
for (final Tag tag : tags) {
if (result.length() > 0) {
result.append(", ");
}
result.append(STRING_COMPRESSOR.get(tag.getKey()));
result.append("=");
result.append(STRING_COMPRESSOR.get(tag.getValue()));
}
result.append(STRING_COMPRESSOR.get(tag.getKey()));
result.append("=");
result.append(STRING_COMPRESSOR.get(tag.getValue()));
}
return result.toString();
}
return result.toString();
}
}

42
pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java
View File

@@ -5,30 +5,30 @@ import java.util.List;
public class TagsBuilder {
final List<Tag> tags = new ArrayList<>();
final List<Tag> tags = new ArrayList<>();
public static TagsBuilder create() {
return new TagsBuilder();
}
public static TagsBuilder create() {
return new TagsBuilder();
}
public TagsBuilder add(final int key, final int value) {
tags.add(new Tag(key, value));
return this;
}
public TagsBuilder add(final int key, final int value) {
tags.add(new Tag(key, value));
return this;
}
public TagsBuilder add(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.getIfPresent(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.getIfPresent(value);
return add(keyAsInt, valueAsInt);
}
public TagsBuilder add(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.getIfPresent(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.getIfPresent(value);
return add(keyAsInt, valueAsInt);
}
public TagsBuilder addAndAddToDictionary(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.put(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.put(value);
return add(keyAsInt, valueAsInt);
}
public TagsBuilder addAndAddToDictionary(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.put(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.put(value);
return add(keyAsInt, valueAsInt);
}
public Tags build() {
return Tags.create(tags);
}
public Tags build() {
return Tags.create(tags);
}
}

288
pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java
View File

@@ -31,182 +31,182 @@ import java.util.regex.Pattern;
* retrievals.
*/
public class UniqueStringIntegerPairs {
private static final String SEPARATOR = "\t";
private static final String SEPARATOR = "\t";
private static final boolean APPEND = true;
private static final boolean APPEND = true;
private static final class ByteArray implements Comparable<ByteArray> {
private final byte[] array;
private final int start;
private final int endExclusive;
private static final class ByteArray implements Comparable<ByteArray> {
private final byte[] array;
private final int start;
private final int endExclusive;
public ByteArray(final byte[] array, final int start, final int endExclusive) {
super();
this.array = array;
this.start = start;
this.endExclusive = endExclusive;
}
public ByteArray(final byte[] array, final int start, final int endExclusive) {
super();
this.array = array;
this.start = start;
this.endExclusive = endExclusive;
}
public ByteArray(final byte[] bytes) {
this.array = bytes;
this.start = 0;
this.endExclusive = bytes.length;
}
public ByteArray(final byte[] bytes) {
this.array = bytes;
this.start = 0;
this.endExclusive = bytes.length;
}
// custom hashcode!
@Override
public int hashCode() {
int result = 1;
final byte[] a = array;
final int end = endExclusive;
for (int i = start; i < end; i++) {
result = 31 * result + a[i];
}
return result;
}
// custom hashcode!
@Override
public int hashCode() {
int result = 1;
final byte[] a = array;
final int end = endExclusive;
for (int i = start; i < end; i++) {
result = 31 * result + a[i];
}
return result;
}
// custom equals!
@Override
public boolean equals(final Object obj) {
final ByteArray other = (ByteArray) obj;
if (!Arrays.equals(array, start, endExclusive, other.array, other.start, other.endExclusive))
return false;
return true;
}
// custom equals!
@Override
public boolean equals(final Object obj) {
final ByteArray other = (ByteArray) obj;
if (!Arrays.equals(array, start, endExclusive, other.array, other.start, other.endExclusive))
return false;
return true;
}
@Override
public int compareTo(final ByteArray o) {
return Arrays.compare(array, start, endExclusive, o.array, o.start, o.endExclusive);
}
@Override
public int compareTo(final ByteArray o) {
return Arrays.compare(array, start, endExclusive, o.array, o.start, o.endExclusive);
}
}
}
/**
* Maps a string to an integer. E.g. "myLongValue" -> 123
*/
private final Map<String, Integer> stringToInt = new HashMap<>();
/**
* Maps a string to an integer. E.g. "myLongValue" -> 123
*/
private final Map<String, Integer> stringToInt = new HashMap<>();
private final Map<ByteArray, Integer> bytesToInt = new HashMap<>();
private final Map<ByteArray, Integer> bytesToInt = new HashMap<>();
/**
* Maps an integer to a string. E.g. 123 -> "myLongValue"
*/
private final List<String> intToString = new ArrayList<>();
/**
* Maps an integer to a string. E.g. 123 -> "myLongValue"
*/
private final List<String> intToString = new ArrayList<>();
private final Path file;
private final Path file;
public UniqueStringIntegerPairs() {
this(null);
}
public UniqueStringIntegerPairs() {
this(null);
}
public UniqueStringIntegerPairs(final Path file) {
this.file = file;
if (file != null) {
init(file);
}
}
public UniqueStringIntegerPairs(final Path file) {
this.file = file;
if (file != null) {
init(file);
}
}
private void init(final Path file) throws RuntimeIOException {
private void init(final Path file) throws RuntimeIOException {
try {
Files.createDirectories(file.getParent());
if (!Files.exists(file)) {
Files.createFile(file);
}
try {
Files.createDirectories(file.getParent());
if (!Files.exists(file)) {
Files.createFile(file);
}
try (final BufferedReader reader = new BufferedReader(
new InputStreamReader(new FileInputStream(file.toFile()), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
try (final BufferedReader reader = new BufferedReader(
new InputStreamReader(new FileInputStream(file.toFile()), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
final String[] tokens = line.split(Pattern.quote(SEPARATOR));
final String[] tokens = line.split(Pattern.quote(SEPARATOR));
if (tokens.length == 2) {
final String string = tokens[0];
final int integer = Integer.parseInt(tokens[1]);
intToStringPut(integer, string);
stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
}
}
}
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
if (tokens.length == 2) {
final String string = tokens[0];
final int integer = Integer.parseInt(tokens[1]);
intToStringPut(integer, string);
stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
}
}
}
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private void intToStringPut(final int value, final String string) {
if (intToString.size() <= value) {
// list is not long enough -> grow list
while (intToString.size() <= value) {
intToString.add(null);
}
}
intToString.set(value, string);
}
private void intToStringPut(final int value, final String string) {
if (intToString.size() <= value) {
// list is not long enough -> grow list
while (intToString.size() <= value) {
intToString.add(null);
}
}
intToString.set(value, string);
}
void put(final String string, final int integer) {
void put(final String string, final int integer) {
if (stringToInt.containsKey(string) || (intToString.size() > integer && intToString.get(integer) != null)) {
throw new IllegalArgumentException("Unique key constraint violation for (" + string + ", " + integer + ")");
}
if (file != null) {
try (final Writer writer = new OutputStreamWriter(new FileOutputStream(file.toFile(), APPEND),
StandardCharsets.UTF_8)) {
if (stringToInt.containsKey(string) || (intToString.size() > integer && intToString.get(integer) != null)) {
throw new IllegalArgumentException("Unique key constraint violation for (" + string + ", " + integer + ")");
}
if (file != null) {
try (final Writer writer = new OutputStreamWriter(new FileOutputStream(file.toFile(), APPEND),
StandardCharsets.UTF_8)) {
writer.write(string + SEPARATOR + integer + "\n");
writer.write(string + SEPARATOR + integer + "\n");
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
intToStringPut(integer, string);
stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
}
intToStringPut(integer, string);
stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
}
public Integer get(final String string) {
public Integer get(final String string) {
return stringToInt.get(string);
}
return stringToInt.get(string);
}
public String getKey(final int second) {
return intToString.get(second);
}
public String getKey(final int second) {
return intToString.get(second);
}
public Integer getHighestInteger() {
return intToString.size() == 0 ? -1 : intToString.size() - 1;
}
public Integer getHighestInteger() {
return intToString.size() == 0 ? -1 : intToString.size() - 1;
}
public Integer computeIfAbsent(final String string, final Function<String, Integer> mappingFunction) {
if (!stringToInt.containsKey(string)) {
synchronized (stringToInt) {
if (!stringToInt.containsKey(string)) {
final Integer second = mappingFunction.apply(string);
put(string, second);
}
}
}
public Integer computeIfAbsent(final String string, final Function<String, Integer> mappingFunction) {
if (!stringToInt.containsKey(string)) {
synchronized (stringToInt) {
if (!stringToInt.containsKey(string)) {
final Integer second = mappingFunction.apply(string);
put(string, second);
}
}
}
return stringToInt.get(string);
}
return stringToInt.get(string);
}
public Integer computeIfAbsent(final byte[] bytes, final int start, final int endExclusive) {
public Integer computeIfAbsent(final byte[] bytes, final int start, final int endExclusive) {
final ByteArray byteArray = new ByteArray(bytes, start, endExclusive);
Integer result = bytesToInt.get(byteArray);
if (result == null) {
synchronized (stringToInt) {
if (!bytesToInt.containsKey(byteArray)) {
final String string = new String(bytes, start, endExclusive - start, StandardCharsets.UTF_8);
final Integer integer = intToString.size();
put(string, integer);
}
result = bytesToInt.get(byteArray);
}
}
final ByteArray byteArray = new ByteArray(bytes, start, endExclusive);
Integer result = bytesToInt.get(byteArray);
if (result == null) {
synchronized (stringToInt) {
if (!bytesToInt.containsKey(byteArray)) {
final String string = new String(bytes, start, endExclusive - start, StandardCharsets.UTF_8);
final Integer integer = intToString.size();
put(string, integer);
}
result = bytesToInt.get(byteArray);
}
}
return result;
}
return result;
}
}

210
pdb-api/src/test/java/org/lucares/memory/MemoryScale.java
View File

@@ -14,133 +14,133 @@ import org.lucares.pdb.api.UniqueStringIntegerPairs;
public class MemoryScale {
public static final String A = "A";
public static final String A = "A";
public static void main(final String[] args) {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
public static void main(final String[] args) {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
scale("singleTag");
scale("tags0");
scale("tags1");
scale("tags2");
scale("tags6");
}
scale("singleTag");
scale("tags0");
scale("tags1");
scale("tags2");
scale("tags6");
}
private static void scale(final String what) {
System.out.println("start: " + what);
// warmup of classes
getUsedMemory();
Object handle = createObject(what);
private static void scale(final String what) {
System.out.println("start: " + what);
// warmup of classes
getUsedMemory();
Object handle = createObject(what);
handle = null;
handle = null;
runGc();
final long memoryBefore = getUsedMemory();
runGc();
final long memoryBefore = getUsedMemory();
handle = createObject(what);
handle = createObject(what);
runGc();
final long memoryAfter = getUsedMemory();
System.out.println(what + ": used memory: " + (memoryAfter - memoryBefore));
handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler
}
runGc();
final long memoryAfter = getUsedMemory();
System.out.println(what + ": used memory: " + (memoryAfter - memoryBefore));
handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler
}
private static Object createObject(final String what) {
private static Object createObject(final String what) {
switch (what) {
case "singleTag":
return createTag();
case "tags0":
return createTags0();
case "tags1":
return createTags1();
case "tags2":
return createTags2();
case "tags6":
return createTags6();
case "string":
return createString();
case "linkedHashMap":
return createLinkedHashMap();
case "path":
return createPath("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsString":
return createPathAsString("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsUtf8":
return createPathAsUtf8("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
default:
return null;
}
}
switch (what) {
case "singleTag":
return createTag();
case "tags0":
return createTags0();
case "tags1":
return createTags1();
case "tags2":
return createTags2();
case "tags6":
return createTags6();
case "string":
return createString();
case "linkedHashMap":
return createLinkedHashMap();
case "path":
return createPath("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsString":
return createPathAsString("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsUtf8":
return createPathAsUtf8("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
default:
return null;
}
}
private static Object createTag() {
return new Tag("", "");
}
private static Object createTag() {
return new Tag("", "");
}
private static Object createTags0() {
return new Tags();
}
private static Object createTags0() {
return new Tags();
}
private static Object createTags1() {
return Tags.createAndAddToDictionary("k1", "v1");
}
private static Object createTags1() {
return Tags.createAndAddToDictionary("k1", "v1");
}
private static Object createTags2() {
return Tags.createAndAddToDictionary("k1", "v1", "k2", "v2");
}
private static Object createTags2() {
return Tags.createAndAddToDictionary("k1", "v1", "k2", "v2");
}
private static Object createTags6() {
TagsBuilder result = TagsBuilder.create();
result = result.add("k1", "v1");
result = result.add("k2", "v2");
result = result.add("k3", "v3");
result = result.add("k4", "v4");
result = result.add("k5", "v5");
result = result.add("k6", "v6");
return result.build();
}
private static Object createTags6() {
TagsBuilder result = TagsBuilder.create();
result = result.add("k1", "v1");
result = result.add("k2", "v2");
result = result.add("k3", "v3");
result = result.add("k4", "v4");
result = result.add("k5", "v5");
result = result.add("k6", "v6");
return result.build();
}
private static Object createPathAsUtf8(final String string) {
return string.getBytes(StandardCharsets.UTF_8);
}
private static Object createPathAsUtf8(final String string) {
return string.getBytes(StandardCharsets.UTF_8);
}
private static String createPathAsString(final String string) {
return string.replace("C", "c");
}
private static String createPathAsString(final String string) {
return string.replace("C", "c");
}
private static Path createPath(final String string) {
return Paths.get(string);
}
private static Path createPath(final String string) {
return Paths.get(string);
}
private static String createString() {
private static String createString() {
final int i = 0;
return "" + i;
}
final int i = 0;
return "" + i;
}
private static Object createLinkedHashMap() {
final Map<String, String> map = new LinkedHashMap<>();
private static Object createLinkedHashMap() {
final Map<String, String> map = new LinkedHashMap<>();
map.put("A", "A");
for (int i = 0; i < 0; i++) {
map.put("" + i, "" + i);
}
map.put("A", "A");
for (int i = 0; i < 0; i++) {
map.put("" + i, "" + i);
}
return map;
}
return map;
}
private static void runGc() {
for (int i = 0; i < 10; i++) {
System.gc();
try {
Thread.sleep(100);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
private static void runGc() {
for (int i = 0; i < 10; i++) {
System.gc();
try {
Thread.sleep(100);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
private static long getUsedMemory() {
return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
}
private static long getUsedMemory() {
return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
}
}

38
pdb-api/src/test/java/org/lucares/pdb/api/DateTimeRangeTest.java
View File

@@ -11,27 +11,27 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
public class DateTimeRangeTest {
@DataProvider
Object[][] providerIntersect() {
final List<Object[]> result = new ArrayList<>();
@DataProvider
Object[][] providerIntersect() {
final List<Object[]> result = new ArrayList<>();
final OffsetDateTime a = Instant.ofEpochMilli(1000).atOffset(ZoneOffset.UTC);
final OffsetDateTime b = Instant.ofEpochMilli(2000).atOffset(ZoneOffset.UTC);
final OffsetDateTime c = Instant.ofEpochMilli(3000).atOffset(ZoneOffset.UTC);
final OffsetDateTime d = Instant.ofEpochMilli(4000).atOffset(ZoneOffset.UTC);
final OffsetDateTime a = Instant.ofEpochMilli(1000).atOffset(ZoneOffset.UTC);
final OffsetDateTime b = Instant.ofEpochMilli(2000).atOffset(ZoneOffset.UTC);
final OffsetDateTime c = Instant.ofEpochMilli(3000).atOffset(ZoneOffset.UTC);
final OffsetDateTime d = Instant.ofEpochMilli(4000).atOffset(ZoneOffset.UTC);
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(c, d), false });
result.add(new Object[] { new DateTimeRange(a, c), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(c, d), false });
result.add(new Object[] { new DateTimeRange(a, c), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(b, d), true });
return result.toArray(new Object[result.size()][]);
}
return result.toArray(new Object[result.size()][]);
}
@Test(dataProvider = "providerIntersect")
public void testIntersect(final DateTimeRange a, final DateTimeRange b, final boolean expected) throws Exception {
Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b);
Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b);
}
@Test(dataProvider = "providerIntersect")
public void testIntersect(final DateTimeRange a, final DateTimeRange b, final boolean expected) throws Exception {
Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b);
Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b);
}
}

88
pdb-api/src/test/java/org/lucares/pdb/api/StringCompressorTest.java
View File

@@ -18,63 +18,63 @@ import org.testng.annotations.Test;
@Test
public class StringCompressorTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testKeyCompressorRoundtrip() throws Exception {
final StringCompressor keyValueCompressor = StringCompressor.create(dataDirectory.resolve("key.csv"));
public void testKeyCompressorRoundtrip() throws Exception {
final StringCompressor keyValueCompressor = StringCompressor.create(dataDirectory.resolve("key.csv"));
final String value = "foo";
final Integer intFoo = keyValueCompressor.put(value);
final String actual = keyValueCompressor.get(intFoo);
final String value = "foo";
final Integer intFoo = keyValueCompressor.put(value);
final String actual = keyValueCompressor.get(intFoo);
Assert.assertEquals(actual, value);
}
Assert.assertEquals(actual, value);
}
public void testKeyCompressorInitialization() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String value = "foo";
{
final StringCompressor keyValueCompressor = StringCompressor.create(database);
public void testKeyCompressorInitialization() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String value = "foo";
{
final StringCompressor keyValueCompressor = StringCompressor.create(database);
keyValueCompressor.put(value);
}
{
final StringCompressor keyValueCompressor = StringCompressor.create(database);
keyValueCompressor.put(value);
}
{
final StringCompressor keyValueCompressor = StringCompressor.create(database);
keyValueCompressor.get(0);
}
keyValueCompressor.get(0);
}
}
}
@Test(invocationCount = 1)
public void testPutConcurrently() throws InterruptedException, ExecutionException {
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs();
final StringCompressor stringCompressor = new StringCompressor(usip);
@Test(invocationCount = 1)
public void testPutConcurrently() throws InterruptedException, ExecutionException {
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs();
final StringCompressor stringCompressor = new StringCompressor(usip);
final ExecutorService pool = Executors.newCachedThreadPool();
final ExecutorService pool = Executors.newCachedThreadPool();
final int numEntries = 1000;
final Future<List<String>> future1 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future2 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future3 = pool.submit(new StringInserter(stringCompressor, numEntries));
final int numEntries = 1000;
final Future<List<String>> future1 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future2 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future3 = pool.submit(new StringInserter(stringCompressor, numEntries));
future1.get();
future2.get();
future3.get();
future1.get();
future2.get();
future3.get();
pool.shutdown();
pool.awaitTermination(1, TimeUnit.MILLISECONDS);
pool.shutdown();
pool.awaitTermination(1, TimeUnit.MILLISECONDS);
Assert.assertEquals((int) usip.getHighestInteger(), 3 * numEntries - 1);
}
Assert.assertEquals((int) usip.getHighestInteger(), 3 * numEntries - 1);
}
}

32
pdb-api/src/test/java/org/lucares/pdb/api/StringInserter.java
View File

@@ -7,23 +7,23 @@ import java.util.concurrent.Callable;
final class StringInserter implements Callable<List<String>> {
private final StringCompressor stringCompressor;
private final int numEntries;
private final StringCompressor stringCompressor;
private final int numEntries;
public StringInserter(final StringCompressor stringCompressor, final int numEntries) {
this.stringCompressor = stringCompressor;
this.numEntries = numEntries;
}
public StringInserter(final StringCompressor stringCompressor, final int numEntries) {
this.stringCompressor = stringCompressor;
this.numEntries = numEntries;
}
@Override
public List<String> call() throws Exception {
@Override
public List<String> call() throws Exception {
final List<String> result = new ArrayList<>();
for (int i = 0; i < numEntries; i++) {
final String s = UUID.randomUUID().toString();
stringCompressor.put(s);
result.add(s);
}
return result;
}
final List<String> result = new ArrayList<>();
for (int i = 0; i < numEntries; i++) {
final String s = UUID.randomUUID().toString();
stringCompressor.put(s);
result.add(s);
}
return result;
}
}

96
pdb-api/src/test/java/org/lucares/pdb/api/UniqueStringIntegerPairsTest.java
View File

@@ -13,62 +13,62 @@ import org.testng.annotations.Test;
@Test
public class UniqueStringIntegerPairsTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testPutGet() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String first = "key1";
final Integer second = 1;
public void testPutGet() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String first = "key1";
final Integer second = 1;
{
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
{
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
usip.put(first, second);
Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first);
}
usip.put(first, second);
Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first);
}
{
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
{
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first);
}
}
Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first);
}
}
public void testUniqueKeyContstraint() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String first = "key1";
final Integer second = 1;
public void testUniqueKeyContstraint() throws Exception {
final Path database = dataDirectory.resolve("key.csv");
final String first = "key1";
final Integer second = 1;
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
usip.put(first, second);
try {
// cannot add another pair with the first key
final int another = second + 1;
usip.put(first, another);
Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) {
// expected
}
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
usip.put(first, second);
try {
// cannot add another pair with the first key
final int another = second + 1;
usip.put(first, another);
Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) {
// expected
}
try {
// cannot add another pair with the same second value
final String another = first + 1;
usip.put(another, second);
Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) {
// expected
}
}
try {
// cannot add another pair with the same second value
final String another = first + 1;
usip.put(another, second);
Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) {
// expected
}
}
}

23
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java
View File

@@ -1,17 +1,18 @@
package org.lucares.pdb.plot.api;
/**
* Note: The order in this enum defines the order in which the aggregates are drawn.
* Note: The order in this enum defines the order in which the aggregates are
* drawn.
*/
public enum Aggregate {
PARALLEL,
SCATTER,
/**
* Empirical cumulative distribution functions
*
* @see https://serialmentor.com/dataviz/ecdf-qq.html
*/
CUM_DISTRIBUTION,
PARALLEL,
SCATTER,
/**
* Empirical cumulative distribution functions
*
* @see https://serialmentor.com/dataviz/ecdf-qq.html
*/
CUM_DISTRIBUTION,
}

77
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java
View File

@@ -13,52 +13,53 @@ import org.lucares.recommind.logs.LineStyle;
public abstract class AggregateHandler implements Appender {
private GnuplotAxis xAxis = GnuplotAxis.X1;
private GnuplotAxis yAxis = GnuplotAxis.Y1;
public GnuplotAxis getxAxis() {
return xAxis;
}
private GnuplotAxis xAxis = GnuplotAxis.X1;
public void updateAxis(GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
this.xAxis = axis;
break;
case Y1:
case Y2:
this.yAxis = axis;
break;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
private GnuplotAxis yAxis = GnuplotAxis.Y1;
public GnuplotAxis getxAxis() {
return xAxis;
}
}
public GnuplotAxis getyAxis() {
return yAxis;
}
public void updateAxis(final GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
this.xAxis = axis;
break;
case Y1:
case Y2:
this.yAxis = axis;
break;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
protected String gnuplotXYAxis() {
return xAxis.getAxisNameForPlots()+yAxis.getAxisNameForPlots();
}
abstract Type getAxisType(GnuplotAxis axis);
public GnuplotAxis getyAxis() {
return yAxis;
}
abstract Aggregate getAggregateType();
protected String gnuplotXYAxis() {
return xAxis.getAxisNameForPlots() + yAxis.getAxisNameForPlots();
}
abstract AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
abstract Type getAxisType(GnuplotAxis axis);
abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
abstract Aggregate getAggregateType();
abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Optional<String> title);
abstract AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli);
abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
protected String gnuplotTitle(Optional<String> title) {
return title.isPresent() ? "title '" + title.get() + "'" : "notitle";
}
abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title);
abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli);
protected String gnuplotTitle(final Optional<String> title) {
return title.isPresent() ? "title '" + title.get() + "'" : "notitle";
}
}

166
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java
View File

@@ -17,91 +17,97 @@ import org.lucares.utils.CollectionUtils;
import org.lucares.utils.Preconditions;
public class AggregateHandlerCollection {
private static final Comparator<AggregateHandler> PLOTTING_ORDER = Comparator.comparing(AggregateHandler::getAggregateType);
private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
private static final Comparator<AggregateHandler> PLOTTING_ORDER = Comparator
.comparing(AggregateHandler::getAggregateType);
public void add(AggregateHandler aggregateHandler) {
aggregateHandlers.add(aggregateHandler);
}
public void updateAxisForHandlers() {
updateAxisForHandlers(GnuplotAxis.X1);
updateAxisForHandlers(GnuplotAxis.Y1);
}
private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
private void updateAxisForHandlers(GnuplotAxis axis) {
final EnumSet<Type> result = EnumSet.noneOf(Type.class);
for (AggregateHandler handler : aggregateHandlers) {
final Type type = handler.getAxisType(axis);
if (result.isEmpty()) {
result.add(type);
}else {
final boolean containsType = result.contains(type);
if (containsType) {
// already has an axis of this type
// TODO merge axis definitions and use the greater values for: range, ticsIncrement
} else{
Preconditions.checkSmaller(result.size(), 2, "at most two different axis are supported");
final GnuplotAxis mirrorAxis = axis.mirrorAxis();
handler.updateAxis(mirrorAxis);
result.add(type);
public void add(final AggregateHandler aggregateHandler) {
aggregateHandlers.add(aggregateHandler);
}
public void updateAxisForHandlers() {
updateAxisForHandlers(GnuplotAxis.X1);
updateAxisForHandlers(GnuplotAxis.Y1);
}
private void updateAxisForHandlers(final GnuplotAxis axis) {
final EnumSet<Type> result = EnumSet.noneOf(Type.class);
for (final AggregateHandler handler : aggregateHandlers) {
final Type type = handler.getAxisType(axis);
if (result.isEmpty()) {
result.add(type);
} else {
final boolean containsType = result.contains(type);
if (containsType) {
// already has an axis of this type
// TODO merge axis definitions and use the greater values for: range,
// ticsIncrement
} else {
Preconditions.checkSmaller(result.size(), 2, "at most two different axis are supported");
final GnuplotAxis mirrorAxis = axis.mirrorAxis();
handler.updateAxis(mirrorAxis);
result.add(type);
}
}
}
}
}
}
public List<AxisSettings> getXAxisDefinitions(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
final List<AxisSettings> result = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
AxisSettings axis = handler.createXAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public List<AxisSettings> getYAxisDefinitions(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
List<AxisSettings> result = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
final AxisSettings axis = handler.createYAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public AggregatorCollection createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli) {
final List<CustomAggregator> aggregators = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
final CustomAggregator aggregator = handler.createCustomAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli);
if (aggregator != null) {
aggregators.add(aggregator);
}
}
return new AggregatorCollection(aggregators);
}
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) {
boolean first = true;
final List<AggregateHandler> handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers, PLOTTING_ORDER);
for (AggregateHandler handler : handlersInPlottingOrder) {
for (DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty();
Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData().get(handler.getAggregateType());
if(aggregatedData.isPresent()) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title);
public List<AxisSettings> getXAxisDefinitions(final GnuplotSettings settings,
final Collection<DataSeries> dataSeries) {
final List<AxisSettings> result = new ArrayList<>();
for (final AggregateHandler handler : aggregateHandlers) {
final AxisSettings axis = handler.createXAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public List<AxisSettings> getYAxisDefinitions(final GnuplotSettings settings,
final Collection<DataSeries> dataSeries) {
final List<AxisSettings> result = new ArrayList<>();
for (final AggregateHandler handler : aggregateHandlers) {
final AxisSettings axis = handler.createYAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public AggregatorCollection createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
final List<CustomAggregator> aggregators = new ArrayList<>();
for (final AggregateHandler handler : aggregateHandlers) {
final CustomAggregator aggregator = handler.createCustomAggregator(tmpDir, plotSettings, fromEpochMilli,
toEpochMilli);
if (aggregator != null) {
aggregators.add(aggregator);
}
}
return new AggregatorCollection(aggregators);
}
public void addPlots(final StringBuilder result, final Collection<DataSeries> dataSeries) {
boolean first = true;
final List<AggregateHandler> handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers,
PLOTTING_ORDER);
for (final AggregateHandler handler : handlersInPlottingOrder) {
for (final DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty();
final Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData()
.get(handler.getAggregateType());
if (aggregatedData.isPresent()) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title);
}
}
first = false;
}
}
first = false;
}
}
}

24
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java
View File

@@ -3,19 +3,19 @@ package org.lucares.pdb.plot.api;
import java.io.File;
public class AggregatedData {
private final String label;
private final File dataFile;
private final String label;
private final File dataFile;
public AggregatedData(final String label, final File dataFile) {
this.label = label;
this.dataFile = dataFile;
}
public AggregatedData(final String label, final File dataFile) {
this.label = label;
this.dataFile = dataFile;
}
public String getLabel() {
return label;
}
public String getLabel() {
return label;
}
public File getDataFile() {
return dataFile;
}
public File getDataFile() {
return dataFile;
}
}

24
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java
View File

@@ -4,19 +4,19 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Optional;
public class AggregatedDataCollection implements Iterable<AggregatedData>{
private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>();
public class AggregatedDataCollection implements Iterable<AggregatedData> {
private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>();
public void put(Aggregate aggregate, AggregatedData aggregatedData) {
aggregatedDatas.put(aggregate, aggregatedData);
}
public void put(final Aggregate aggregate, final AggregatedData aggregatedData) {
aggregatedDatas.put(aggregate, aggregatedData);
}
@Override
public Iterator<AggregatedData> iterator() {
return aggregatedDatas.values().iterator();
}
@Override
public Iterator<AggregatedData> iterator() {
return aggregatedDatas.values().iterator();
}
public Optional<AggregatedData> get(Aggregate aggregateType) {
return Optional.ofNullable(aggregatedDatas.get(aggregateType));
}
public Optional<AggregatedData> get(final Aggregate aggregateType) {
return Optional.ofNullable(aggregatedDatas.get(aggregateType));
}
}

29
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataEntry.java
View File

@@ -1,18 +1,21 @@
package org.lucares.pdb.plot.api;
public class AggregatedDataEntry {
private final double epochSeconds;
private final long value;
public AggregatedDataEntry(double epochSeconds, long value) {
super();
this.epochSeconds = epochSeconds;
this.value = value;
}
public double getEpochSeconds() {
return epochSeconds;
}
public long getValue() {
return value;
}
private final double epochSeconds;
private final long value;
public AggregatedDataEntry(final double epochSeconds, final long value) {
super();
this.epochSeconds = epochSeconds;
this.value = value;
}
public double getEpochSeconds() {
return epochSeconds;
}
public long getValue() {
return value;
}
}

36
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java
View File

@@ -4,26 +4,26 @@ import java.io.IOException;
import java.util.List;
public class AggregatorCollection {
private final List<CustomAggregator> aggregators;
private final List<CustomAggregator> aggregators;
public AggregatorCollection(List<CustomAggregator> aggregators) {
this.aggregators = aggregators;
}
public void addValue(boolean valueIsInYRange, long epochMilli, long value) {
for (CustomAggregator aggregator : aggregators) {
aggregator.addValue(valueIsInYRange, epochMilli, value);
public AggregatorCollection(final List<CustomAggregator> aggregators) {
this.aggregators = aggregators;
}
}
public AggregatedDataCollection getAggregatedData() throws IOException {
AggregatedDataCollection result = new AggregatedDataCollection();
for (CustomAggregator aggregator : aggregators) {
result.put(aggregator.getType(), aggregator.getAggregatedData());
public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
for (final CustomAggregator aggregator : aggregators) {
aggregator.addValue(valueIsInYRange, epochMilli, value);
}
}
public AggregatedDataCollection getAggregatedData() throws IOException {
final AggregatedDataCollection result = new AggregatedDataCollection();
for (final CustomAggregator aggregator : aggregators) {
result.put(aggregator.getType(), aggregator.getAggregatedData());
}
return result;
}
return result;
}
}

18
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java
View File

@@ -3,15 +3,15 @@ package org.lucares.pdb.plot.api;
import java.util.Locale;
public interface Appender {
default void appendln(final StringBuilder builder, final String string) {
builder.append(string + "\n");
}
default void appendln(final StringBuilder builder, final String string) {
builder.append(string + "\n");
}
default void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US,format + "\n", args));
}
default void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US, format + "\n", args));
}
default void appendf(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US,format, args));
}
default void appendf(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US, format, args));
}
}

2
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AxisScale.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.plot.api;
public enum AxisScale {
LINEAR, LOG10
LINEAR, LOG10
}

162
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java
View File

@@ -15,114 +15,114 @@ import org.lucares.collections.LongLongHashMap;
public class CumulativeDistributionCustomAggregator implements CustomAggregator {
private final static int POINTS = 500;
private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer {
private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0;
private long cumulativeCount = 0;
private long maxValue = 0;
private long maxValue = 0;
private final LinkedHashMap<Double, Long> percentiles = new LinkedHashMap<>(POINTS);
private final LinkedHashMap<Double, Long> percentiles = new LinkedHashMap<>(POINTS);
private final double stepSize;
private final double stepSize;
private double lastPercentile;
private double nextPercentile;
private double lastPercentile;
private double nextPercentile;
private final long totalValues;
private final long totalValues;
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
public ToPercentiles(final long totalValues) {
this.totalValues = totalValues;
stepSize = 100.0 / POINTS;
nextPercentile = stepSize;
}
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
@Override
public void accept(final long duration, final long count) {
maxValue = duration;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
percentiles.put(currentPercentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) {
percentiles.put(currentPercentile, duration);
currentPercentile += stepSize;
}
nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize;
}
}
public long getMaxValue() {
return maxValue;
}
public long getMaxValue() {
return maxValue;
}
public LinkedHashMap<Double, Long> getPercentiles() {
return percentiles;
}
public LinkedHashMap<Double, Long> getPercentiles() {
return percentiles;
}
}
}
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
// the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0;
private long totalValues = 0;
private final Path tmpDir;
private final Path tmpDir;
public CumulativeDistributionCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir;
}
public CumulativeDistributionCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir;
}
@Override
public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) {
map.compute(value, 0, l -> l + 1);
totalValues++;
}
@Override
public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) {
map.compute(value, 0, l -> l + 1);
totalValues++;
}
@Override
public AggregatedData getAggregatedData() throws IOException {
final char separator = ',';
final char newline = '\n';
@Override
public AggregatedData getAggregatedData() throws IOException {
final char separator = ',';
final char newline = '\n';
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
map.forEachOrdered(toPercentiles);
final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
map.forEachOrdered(toPercentiles);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder();
if (map.size() > 0) {
// compute the percentiles
toPercentiles.getPercentiles().forEach((percentile, value) -> {
final StringBuilder data = new StringBuilder();
if (map.size() > 0) {
// compute the percentiles
toPercentiles.getPercentiles().forEach((percentile, value) -> {
data.append(percentile);
data.append(separator);
data.append(value);
data.append(newline);
});
data.append(percentile);
data.append(separator);
data.append(value);
data.append(newline);
});
final long maxValue = toPercentiles.getMaxValue();
data.append(100);
data.append(separator);
data.append(maxValue);
data.append(newline);
}
output.write(data.toString());
final long maxValue = toPercentiles.getMaxValue();
data.append(100);
data.append(separator);
data.append(maxValue);
data.append(newline);
}
output.write(data.toString());
}
}
final String title = String.format("cumulative distribution");
return new AggregatedData(title, dataFile);
}
final String title = String.format("cumulative distribution");
return new AggregatedData(title, dataFile);
}
@Override
public Aggregate getType() {
return Aggregate.CUM_DISTRIBUTION;
}
@Override
public Aggregate getType() {
return Aggregate.CUM_DISTRIBUTION;
}
}

128
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java
View File

@@ -14,74 +14,74 @@ import org.lucares.recommind.logs.AxisSettings.Type;
public class CumulativeDistributionHandler extends AggregateHandler {
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir);
}
public CumulativeDistributionHandler() {
}
@Override
Type getAxisType(GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
return Type.Percent;
case Y1:
case Y2:
return Type.Duration;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
@Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir);
}
}
@Override
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = new AxisSettings();
result.setLabel("Cumulative Distribution");
result.setType(Type.Percent);
result.setAxis(getxAxis());
result.setFormat("%.0f%%");
result.setTicIncrement(computeTicIncrement(settings));
result.setFrom("0");
result.setTo("100");
return result;
}
private int computeTicIncrement(GnuplotSettings settings) {
int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
if (widthByFontSize < 50) {
return 20;
} else if (widthByFontSize < 75) {
return 10;
} else {
return 5;
public CumulativeDistributionHandler() {
}
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
gnuplotXYAxis(), //
lineStyle.darker()//
);
}
@Override
Type getAxisType(GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
return Type.Percent;
case Y1:
case Y2:
return Type.Duration;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
@Override
public Aggregate getAggregateType() {
return Aggregate.CUM_DISTRIBUTION;
}
@Override
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = new AxisSettings();
result.setLabel("Cumulative Distribution");
result.setType(Type.Percent);
result.setAxis(getxAxis());
result.setFormat("%.0f%%");
result.setTicIncrement(computeTicIncrement(settings));
result.setFrom("0");
result.setTo("100");
return result;
}
private int computeTicIncrement(GnuplotSettings settings) {
int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
if (widthByFontSize < 50) {
return 20;
} else if (widthByFontSize < 75) {
return 10;
} else {
return 5;
}
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
gnuplotXYAxis(), //
lineStyle.darker()//
);
}
@Override
public Aggregate getAggregateType() {
return Aggregate.CUM_DISTRIBUTION;
}
}

Some files were not shown because too many files have changed in this diff Show More