andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

apply new code formatter and save action

Browse Source
This commit is contained in:
Andreas Huber
2019-11-24 10:20:43 +01:00
parent 5ea82c6a4c
commit 06b379494f
184 changed files with 13455 additions and 13489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFile.java
View File

@@ -37,212 +37,213 @@ import org.slf4j.LoggerFactory;
*/ */
public class BSFile implements AutoCloseable { public class BSFile implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(BSFile.class); private static final Logger LOGGER = LoggerFactory.getLogger(BSFile.class);
public static final int BLOCK_SIZE = 512; public static final int BLOCK_SIZE = 512;
/* /*
* The last disk block of this sequence. This is the block new values will be * The last disk block of this sequence. This is the block new values will be
* appended to. * appended to.
*/ */
private BSFileDiskBlock buffer; private BSFileDiskBlock buffer;
private int offsetInBuffer = 0; private int offsetInBuffer = 0;
private boolean dirty = false; private boolean dirty = false;
private final long rootBlockOffset; private final long rootBlockOffset;
private final DiskStorage diskStorage; private final DiskStorage diskStorage;
private final BSFileDiskBlock rootDiskBlock; private final BSFileDiskBlock rootDiskBlock;
private final BSFileCustomizer customizer; private final BSFileCustomizer customizer;
BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer) { BSFile(final long rootBlockOffset, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this(new BSFileDiskBlock(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE)), diskStorage, customizer); this(new BSFileDiskBlock(diskStorage.getDiskBlock(rootBlockOffset, BLOCK_SIZE)), diskStorage, customizer);
} }
BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer) { BSFile(final BSFileDiskBlock rootDiskBlock, final DiskStorage diskStorage, final BSFileCustomizer customizer) {
this.rootDiskBlock = rootDiskBlock; this.rootDiskBlock = rootDiskBlock;
this.customizer = customizer; this.customizer = customizer;
this.rootBlockOffset = rootDiskBlock.getBlockOffset(); this.rootBlockOffset = rootDiskBlock.getBlockOffset();
this.diskStorage = diskStorage; this.diskStorage = diskStorage;
final long lastBlockNumber = rootDiskBlock.getLastBlockPointer(); final long lastBlockNumber = rootDiskBlock.getLastBlockPointer();
if (lastBlockNumber == rootBlockOffset || lastBlockNumber == 0) { if (lastBlockNumber == rootBlockOffset || lastBlockNumber == 0) {
buffer = rootDiskBlock; buffer = rootDiskBlock;
} else { } else {
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(lastBlockNumber, BLOCK_SIZE)); buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(lastBlockNumber, BLOCK_SIZE));
} }
offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer); offsetInBuffer = determineWriteOffsetInExistingBuffer(buffer);
customizer.init(buffer); customizer.init(buffer);
LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockOffset, lastBlockNumber); LOGGER.trace("create bsFile={} lastBlockNumber={}", rootBlockOffset, lastBlockNumber);
} }
private int determineWriteOffsetInExistingBuffer(final BSFileDiskBlock buffer) { private int determineWriteOffsetInExistingBuffer(final BSFileDiskBlock buffer) {
final byte[] buf = buffer.getBuffer(); final byte[] buf = buffer.getBuffer();
int result = 0; int result = 0;
while (result < buf.length && buf[result] != 0) { while (result < buf.length && buf[result] != 0) {
result++; result++;
} }
return result; return result;
} }
public static BSFile existingFile(final long blockNumber, final DiskStorage diskStorage, public static BSFile existingFile(final long blockNumber, final DiskStorage diskStorage,
final BSFileCustomizer customizer) { final BSFileCustomizer customizer) {
return new BSFile(blockNumber, diskStorage, customizer); return new BSFile(blockNumber, diskStorage, customizer);
} }
public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) { public static BSFile newFile(final DiskStorage diskStorage, final BSFileCustomizer customizer) {
final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE); final long rootBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
LOGGER.trace("create new bsFile={}", rootBlockOffset); LOGGER.trace("create new bsFile={}", rootBlockOffset);
return new BSFile(rootBlockOffset, diskStorage, customizer); return new BSFile(rootBlockOffset, diskStorage, customizer);
} }
public void append(final long value1, final long value2) { public void append(final long value1, final long value2) {
final long val1 = customizer.preProcessWriteValue1(value1); final long val1 = customizer.preProcessWriteValue1(value1);
final long val2 = customizer.preProcessWriteValue2(value2); final long val2 = customizer.preProcessWriteValue2(value2);
final int bytesWritten = VariableByteEncoder.encodeInto(val1, val2, buffer.getBuffer(), offsetInBuffer); final int bytesWritten = VariableByteEncoder.encodeInto(val1, val2, buffer.getBuffer(), offsetInBuffer);
if (bytesWritten == 0) { if (bytesWritten == 0) {
flushFullBufferAndCreateNew(); flushFullBufferAndCreateNew();
customizer.newBlock(); customizer.newBlock();
append(value1, value2); append(value1, value2);
} }
offsetInBuffer += bytesWritten; offsetInBuffer += bytesWritten;
dirty = true; dirty = true;
} }
public void append(final long value) { public void append(final long value) {
int bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer); int bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
if (bytesWritten == 0) { if (bytesWritten == 0) {
flushFullBufferAndCreateNew(); flushFullBufferAndCreateNew();
bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer); bytesWritten = VariableByteEncoder.encodeInto(value, buffer.getBuffer(), offsetInBuffer);
assert bytesWritten > 0 : "after a flush the buffer is emtpy, so it should be possible to write a few bytes"; assert bytesWritten > 0 : "after a flush the buffer is emtpy, so it should be possible to write a few bytes";
} }
offsetInBuffer += bytesWritten; offsetInBuffer += bytesWritten;
dirty = true; dirty = true;
} }
private void flushFullBufferAndCreateNew() { private void flushFullBufferAndCreateNew() {
final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE); final long newBlockOffset = diskStorage.allocateBlock(BLOCK_SIZE);
if (buffer == rootDiskBlock) { if (buffer == rootDiskBlock) {
// root block and current block are the same, so we need // root block and current block are the same, so we need
// to update only one // to update only one
buffer.setLastBlockOffset(newBlockOffset); buffer.setLastBlockOffset(newBlockOffset);
buffer.setNextBlockOffset(newBlockOffset); buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync(); buffer.writeAsync();
} else { } else {
rootDiskBlock.writeLastBlockOffset(newBlockOffset); rootDiskBlock.writeLastBlockOffset(newBlockOffset);
buffer.setNextBlockOffset(newBlockOffset); buffer.setNextBlockOffset(newBlockOffset);
buffer.writeAsync(); buffer.writeAsync();
} }
// set the new buffer // set the new buffer
buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(newBlockOffset, BLOCK_SIZE)); buffer = new BSFileDiskBlock(diskStorage.getDiskBlock(newBlockOffset, BLOCK_SIZE));
offsetInBuffer = 0; offsetInBuffer = 0;
dirty = false; dirty = false;
LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}", rootBlockOffset, newBlockOffset); LOGGER.trace("flushFullBufferAndCreateNew bsFile={} newBlock={}", rootBlockOffset, newBlockOffset);
} }
public void flush() { public void flush() {
LOGGER.trace("flush bsFile={} dirty={} file={}", rootBlockOffset, dirty, diskStorage.getRelativeDatabaseFileForLogging()); LOGGER.trace("flush bsFile={} dirty={} file={}", rootBlockOffset, dirty,
if (dirty) { diskStorage.getRelativeDatabaseFileForLogging());
buffer.writeAsync(); if (dirty) {
} buffer.writeAsync();
} }
}
public Optional<Long> getLastValue() { public Optional<Long> getLastValue() {
final byte[] buf = buffer.getBuffer(); final byte[] buf = buffer.getBuffer();
final LongList bufferedLongs = VariableByteEncoder.decode(buf); final LongList bufferedLongs = VariableByteEncoder.decode(buf);
final Optional<Long> result; final Optional<Long> result;
if (bufferedLongs.isEmpty()) { if (bufferedLongs.isEmpty()) {
result = Optional.empty(); result = Optional.empty();
} else { } else {
final long lastValue = bufferedLongs.get(bufferedLongs.size() - 1); final long lastValue = bufferedLongs.get(bufferedLongs.size() - 1);
result = Optional.of(lastValue); result = Optional.of(lastValue);
} }
return result; return result;
} }
public Stream<LongList> streamOfLongLists() { public Stream<LongList> streamOfLongLists() {
final Iterator<LongList> iterator = new LongListIterator(rootBlockOffset, diskStorage); final Iterator<LongList> iterator = new LongListIterator(rootBlockOffset, diskStorage);
final Stream<LongList> stream = StreamSupport final Stream<LongList> stream = StreamSupport
.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false); .stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false);
final Optional<Function<LongList, LongList>> mapper = customizer.getStreamMapper(); final Optional<Function<LongList, LongList>> mapper = customizer.getStreamMapper();
if (mapper.isPresent()) { if (mapper.isPresent()) {
return stream.map(mapper.get()); return stream.map(mapper.get());
} }
return stream; return stream;
} }
private static class LongListIterator implements Iterator<LongList> { private static class LongListIterator implements Iterator<LongList> {
private LongList next = null; private LongList next = null;
private long nextBlockOffset; private long nextBlockOffset;
private final DiskStorage diskStorage; private final DiskStorage diskStorage;
public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) { public LongListIterator(final long nextBlockNumber, final DiskStorage diskStorage) {
this.nextBlockOffset = nextBlockNumber; this.nextBlockOffset = nextBlockNumber;
this.diskStorage = diskStorage; this.diskStorage = diskStorage;
} }
@Override @Override
public boolean hasNext() { public boolean hasNext() {
return nextBlockOffset != BSFileDiskBlock.NO_NEXT_POINTER; return nextBlockOffset != BSFileDiskBlock.NO_NEXT_POINTER;
} }
@Override @Override
public LongList next() { public LongList next() {
if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) { if (nextBlockOffset == BSFileDiskBlock.NO_NEXT_POINTER) {
throw new NoSuchElementException(); throw new NoSuchElementException();
} }
final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset); final BSFileDiskBlock diskBlock = getDiskBlock(nextBlockOffset);
nextBlockOffset = diskBlock.getNextBlockNumber(); nextBlockOffset = diskBlock.getNextBlockNumber();
final byte[] buf = diskBlock.getBuffer(); final byte[] buf = diskBlock.getBuffer();
next = VariableByteEncoder.decode(buf); next = VariableByteEncoder.decode(buf);
return next; return next;
} }
private BSFileDiskBlock getDiskBlock(final long blockOffset) { private BSFileDiskBlock getDiskBlock(final long blockOffset) {
final DiskBlock diskBlock = diskStorage.getDiskBlock(blockOffset, BLOCK_SIZE); final DiskBlock diskBlock = diskStorage.getDiskBlock(blockOffset, BLOCK_SIZE);
return new BSFileDiskBlock(diskBlock); return new BSFileDiskBlock(diskBlock);
} }
} }
public LongList asLongList() { public LongList asLongList() {
final LongList result = new LongList(); final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll); streamOfLongLists().forEachOrdered(result::addAll);
return result; return result;
} }
public long getRootBlockOffset() { public long getRootBlockOffset() {
return rootBlockOffset; return rootBlockOffset;
} }
@Override @Override
public void close() { public void close() {
flush(); flush();
} }
} }

10
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFileCustomizer.java
View File

@@ -6,13 +6,13 @@ import java.util.function.Function;
import org.lucares.collections.LongList; import org.lucares.collections.LongList;
public interface BSFileCustomizer { public interface BSFileCustomizer {
void init(BSFileDiskBlock lastDiskBlockOfStream); void init(BSFileDiskBlock lastDiskBlockOfStream);
Optional<Function<LongList, LongList>> getStreamMapper(); Optional<Function<LongList, LongList>> getStreamMapper();
void newBlock(); void newBlock();
long preProcessWriteValue1(long value); long preProcessWriteValue1(long value);
long preProcessWriteValue2(long value); long preProcessWriteValue2(long value);
} }

134
block-storage/src/main/java/org/lucares/pdb/blockstorage/BSFileDiskBlock.java
View File

@@ -8,90 +8,90 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
class BSFileDiskBlock { class BSFileDiskBlock {
protected static final int NEXT_POINTER_OFFSET = 0; protected static final int NEXT_POINTER_OFFSET = 0;
public static final long NO_NEXT_POINTER = 0; public static final long NO_NEXT_POINTER = 0;
private static final int LAST_BLOCK_POINTER_POSITION = 8; private static final int LAST_BLOCK_POINTER_POSITION = 8;
public static final long NO_LAST_BLOCK = 0; public static final long NO_LAST_BLOCK = 0;
private static final int INT_SEQUENCE_OFFSET = 8 // next block pointer private static final int INT_SEQUENCE_OFFSET = 8 // next block pointer
+ 8; // last block pointer; + 8; // last block pointer;
private final DiskBlock diskBlock; private final DiskBlock diskBlock;
private long nextBlockOffset = 0; private long nextBlockOffset = 0;
private long lastBlockOffset = 0; private long lastBlockOffset = 0;
private byte[] buffer = null; private byte[] buffer = null;
public BSFileDiskBlock(final DiskBlock diskBlock) { public BSFileDiskBlock(final DiskBlock diskBlock) {
this.diskBlock = diskBlock; this.diskBlock = diskBlock;
} }
public byte[] getBuffer() { public byte[] getBuffer() {
if (buffer == null) { if (buffer == null) {
final ByteBuffer byteBuffer = diskBlock.getByteBuffer(); final ByteBuffer byteBuffer = diskBlock.getByteBuffer();
this.buffer = new byte[byteBuffer.capacity() - INT_SEQUENCE_OFFSET]; this.buffer = new byte[byteBuffer.capacity() - INT_SEQUENCE_OFFSET];
byteBuffer.position(INT_SEQUENCE_OFFSET); byteBuffer.position(INT_SEQUENCE_OFFSET);
byteBuffer.get(buffer); byteBuffer.get(buffer);
} }
return buffer; return buffer;
} }
public long getBlockOffset() { public long getBlockOffset() {
return diskBlock.getBlockOffset(); return diskBlock.getBlockOffset();
} }
public void setNextBlockOffset(final long nextBlockOffset) { public void setNextBlockOffset(final long nextBlockOffset) {
this.nextBlockOffset = nextBlockOffset; this.nextBlockOffset = nextBlockOffset;
} }
public long getLastBlockPointer() { public long getLastBlockPointer() {
if (lastBlockOffset <= 0) { if (lastBlockOffset <= 0) {
lastBlockOffset = diskBlock.getByteBuffer().getLong(LAST_BLOCK_POINTER_POSITION); lastBlockOffset = diskBlock.getByteBuffer().getLong(LAST_BLOCK_POINTER_POSITION);
} }
return lastBlockOffset; return lastBlockOffset;
} }
public long getNextBlockNumber() { public long getNextBlockNumber() {
if (nextBlockOffset <= 0) { if (nextBlockOffset <= 0) {
nextBlockOffset = diskBlock.getByteBuffer().getLong(NEXT_POINTER_OFFSET); nextBlockOffset = diskBlock.getByteBuffer().getLong(NEXT_POINTER_OFFSET);
} }
return nextBlockOffset; return nextBlockOffset;
} }
public void setLastBlockOffset(final long lastBlockOffset) { public void setLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset; this.lastBlockOffset = lastBlockOffset;
} }
public void writeLastBlockOffset(final long lastBlockOffset) { public void writeLastBlockOffset(final long lastBlockOffset) {
this.lastBlockOffset = lastBlockOffset; this.lastBlockOffset = lastBlockOffset;
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset); diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
} }
private void writeBufferToByteBuffer() { private void writeBufferToByteBuffer() {
diskBlock.getByteBuffer().position(INT_SEQUENCE_OFFSET); diskBlock.getByteBuffer().position(INT_SEQUENCE_OFFSET);
diskBlock.getByteBuffer().put(buffer); diskBlock.getByteBuffer().put(buffer);
} }
private void writeBlockHeader() { private void writeBlockHeader() {
diskBlock.getByteBuffer().putLong(NEXT_POINTER_OFFSET, nextBlockOffset); diskBlock.getByteBuffer().putLong(NEXT_POINTER_OFFSET, nextBlockOffset);
diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset); diskBlock.getByteBuffer().putLong(LAST_BLOCK_POINTER_POSITION, lastBlockOffset);
} }
public void writeAsync() { public void writeAsync() {
writeBlockHeader(); writeBlockHeader();
writeBufferToByteBuffer(); writeBufferToByteBuffer();
} }
public void force() { public void force() {
diskBlock.force(); diskBlock.force();
} }
@Override @Override
public String toString() { public String toString() {
final LongList bufferDecoded = VariableByteEncoder.decode(buffer); final LongList bufferDecoded = VariableByteEncoder.decode(buffer);
return "BSFileDiskBlock[bufferDecoded=" + bufferDecoded + "]"; return "BSFileDiskBlock[bufferDecoded=" + bufferDecoded + "]";
} }
} }

56
block-storage/src/main/java/org/lucares/pdb/blockstorage/LongStreamFile.java
View File

@@ -8,41 +8,41 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class LongStreamFile implements AutoCloseable { public class LongStreamFile implements AutoCloseable {
private final BSFile bsFile; private final BSFile bsFile;
LongStreamFile(final BSFile bsFile) { LongStreamFile(final BSFile bsFile) {
this.bsFile = bsFile; this.bsFile = bsFile;
} }
public static LongStreamFile existingFile(final long blockNumber, final DiskStorage diskStorage) public static LongStreamFile existingFile(final long blockNumber, final DiskStorage diskStorage)
throws IOException { throws IOException {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, NullCustomizer.INSTANCE); final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile); return new LongStreamFile(bsFile);
} }
public static LongStreamFile newFile(final DiskStorage diskStorage) throws IOException { public static LongStreamFile newFile(final DiskStorage diskStorage) throws IOException {
final BSFile bsFile = BSFile.newFile(diskStorage, NullCustomizer.INSTANCE); final BSFile bsFile = BSFile.newFile(diskStorage, NullCustomizer.INSTANCE);
return new LongStreamFile(bsFile); return new LongStreamFile(bsFile);
} }
public void append(final long value) throws IOException { public void append(final long value) throws IOException {
bsFile.append(value); bsFile.append(value);
} }
public Stream<LongList> streamOfLongLists() { public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists(); return bsFile.streamOfLongLists();
} }
public LongList asLongList() { public LongList asLongList() {
final LongList result = new LongList(); final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll); streamOfLongLists().forEachOrdered(result::addAll);
return result; return result;
} }
@Override @Override
public void close() { public void close() {
bsFile.close(); bsFile.close();
} }
} }

44
block-storage/src/main/java/org/lucares/pdb/blockstorage/NullCustomizer.java
View File

@@ -7,31 +7,31 @@ import org.lucares.collections.LongList;
public class NullCustomizer implements BSFileCustomizer { public class NullCustomizer implements BSFileCustomizer {
public static final NullCustomizer INSTANCE = new NullCustomizer(); public static final NullCustomizer INSTANCE = new NullCustomizer();
@Override @Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) { public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
// nothing to do - this is a NullObject // nothing to do - this is a NullObject
} }
@Override @Override
public Optional<Function<LongList, LongList>> getStreamMapper() { public Optional<Function<LongList, LongList>> getStreamMapper() {
// no mapper to return - this is a NullObject // no mapper to return - this is a NullObject
return Optional.empty(); return Optional.empty();
} }
@Override @Override
public void newBlock() { public void newBlock() {
// nothing to do - this is a NullObject // nothing to do - this is a NullObject
} }
@Override @Override
public long preProcessWriteValue1(final long value) { public long preProcessWriteValue1(final long value) {
return value; return value;
} }
@Override @Override
public long preProcessWriteValue2(final long value) { public long preProcessWriteValue2(final long value) {
return value; return value;
} }
} }

110
block-storage/src/main/java/org/lucares/pdb/blockstorage/TimeSeriesCustomizer.java
View File

@@ -8,71 +8,71 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
public class TimeSeriesCustomizer implements BSFileCustomizer { public class TimeSeriesCustomizer implements BSFileCustomizer {
private static class TimeStampDeltaDecoder implements Function<LongList, LongList> { private static class TimeStampDeltaDecoder implements Function<LongList, LongList> {
/** /**
* Computes the inverse of the delta encoding in {@link BSFile#appendTimeValue} * Computes the inverse of the delta encoding in {@link BSFile#appendTimeValue}
*/ */
@Override @Override
public LongList apply(final LongList t) { public LongList apply(final LongList t) {
long lastTimeValue = 0; long lastTimeValue = 0;
for (int i = 0; i < t.size(); i += 2) { for (int i = 0; i < t.size(); i += 2) {
lastTimeValue += t.get(i); lastTimeValue += t.get(i);
t.set(i, lastTimeValue); t.set(i, lastTimeValue);
} }
return t; return t;
} }
} }
private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder(); private static final TimeStampDeltaDecoder TIME_DELTA_DECODER = new TimeStampDeltaDecoder();
private long lastEpochMilli; private long lastEpochMilli;
@Override @Override
public void init(final BSFileDiskBlock lastDiskBlockOfStream) { public void init(final BSFileDiskBlock lastDiskBlockOfStream) {
lastEpochMilli = determineLastEpochMilli(lastDiskBlockOfStream); lastEpochMilli = determineLastEpochMilli(lastDiskBlockOfStream);
} }
private long determineLastEpochMilli(final BSFileDiskBlock diskBlock) { private long determineLastEpochMilli(final BSFileDiskBlock diskBlock) {
// get the time/value delta encoded longs // get the time/value delta encoded longs
final byte[] buf = diskBlock.getBuffer(); final byte[] buf = diskBlock.getBuffer();
LongList longList = VariableByteEncoder.decode(buf); LongList longList = VariableByteEncoder.decode(buf);
final long result; final long result;
if (longList.size() < 2) { if (longList.size() < 2) {
// only new files have empty disk blocks // only new files have empty disk blocks
// and empty disk blocks have time offset 0 // and empty disk blocks have time offset 0
result = 0; result = 0;
} else { } else {
// decode the deltas to get the correct timestamps // decode the deltas to get the correct timestamps
longList = TIME_DELTA_DECODER.apply(longList); longList = TIME_DELTA_DECODER.apply(longList);
// return the last timestamp // return the last timestamp
result = longList.get(longList.size() - 2); result = longList.get(longList.size() - 2);
} }
return result; return result;
} }
@Override @Override
public Optional<Function<LongList, LongList>> getStreamMapper() { public Optional<Function<LongList, LongList>> getStreamMapper() {
return Optional.of(TIME_DELTA_DECODER); return Optional.of(TIME_DELTA_DECODER);
} }
@Override @Override
public void newBlock() { public void newBlock() {
lastEpochMilli = 0; lastEpochMilli = 0;
} }
@Override @Override
public long preProcessWriteValue1(final long epochMilli) { public long preProcessWriteValue1(final long epochMilli) {
final long epochMilliDelta = epochMilli - lastEpochMilli; final long epochMilliDelta = epochMilli - lastEpochMilli;
lastEpochMilli = epochMilli; lastEpochMilli = epochMilli;
return epochMilliDelta; return epochMilliDelta;
} }
@Override @Override
public long preProcessWriteValue2(final long value) { public long preProcessWriteValue2(final long value) {
return value; return value;
} }
} }

72
block-storage/src/main/java/org/lucares/pdb/blockstorage/TimeSeriesFile.java
View File

@@ -8,52 +8,52 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class TimeSeriesFile implements AutoCloseable { public class TimeSeriesFile implements AutoCloseable {
private final BSFile bsFile; private final BSFile bsFile;
private TimeSeriesFile(final BSFile bsFile) { private TimeSeriesFile(final BSFile bsFile) {
this.bsFile = bsFile; this.bsFile = bsFile;
} }
public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage) { public static TimeSeriesFile existingFile(final long blockNumber, final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, new TimeSeriesCustomizer()); final BSFile bsFile = BSFile.existingFile(blockNumber, diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile); return new TimeSeriesFile(bsFile);
} }
public static TimeSeriesFile newFile(final DiskStorage diskStorage) { public static TimeSeriesFile newFile(final DiskStorage diskStorage) {
final BSFile bsFile = BSFile.newFile(diskStorage, new TimeSeriesCustomizer()); final BSFile bsFile = BSFile.newFile(diskStorage, new TimeSeriesCustomizer());
return new TimeSeriesFile(bsFile); return new TimeSeriesFile(bsFile);
} }
public void appendTimeValue(final long epochMilli, final long value) { public void appendTimeValue(final long epochMilli, final long value) {
bsFile.append(epochMilli, value); bsFile.append(epochMilli, value);
} }
public Stream<LongList> streamOfLongLists() { public Stream<LongList> streamOfLongLists() {
return bsFile.streamOfLongLists(); return bsFile.streamOfLongLists();
} }
public LongList asTimeValueLongList() { public LongList asTimeValueLongList() {
final LongList result = new LongList(); final LongList result = new LongList();
streamOfLongLists().forEachOrdered(result::addAll); streamOfLongLists().forEachOrdered(result::addAll);
return result; return result;
} }
@Override @Override
public void close() { public void close() {
bsFile.close(); bsFile.close();
} }
public long getRootBlockOffset() { public long getRootBlockOffset() {
return bsFile.getRootBlockOffset(); return bsFile.getRootBlockOffset();
} }
public Optional<Long> getLastValue() { public Optional<Long> getLastValue() {
return bsFile.getLastValue(); return bsFile.getLastValue();
} }
public void flush() { public void flush() {
bsFile.flush(); bsFile.flush();
} }
} }

74
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskBlock.java
View File

@@ -5,52 +5,52 @@ import java.nio.MappedByteBuffer;
public class DiskBlock { public class DiskBlock {
private byte[] buffer = null; private byte[] buffer = null;
private final long blockOffset; private final long blockOffset;
private final ByteBuffer byteBuffer; private final ByteBuffer byteBuffer;
public DiskBlock(final long blockOffset, final ByteBuffer byteBuffer) { public DiskBlock(final long blockOffset, final ByteBuffer byteBuffer) {
this.blockOffset = blockOffset; this.blockOffset = blockOffset;
this.byteBuffer = byteBuffer; this.byteBuffer = byteBuffer;
} }
public byte[] getBuffer() { public byte[] getBuffer() {
if (buffer == null) { if (buffer == null) {
this.buffer = new byte[byteBuffer.capacity()]; this.buffer = new byte[byteBuffer.capacity()];
byteBuffer.get(buffer); byteBuffer.get(buffer);
} }
return buffer; return buffer;
} }
public ByteBuffer getByteBuffer() { public ByteBuffer getByteBuffer() {
return byteBuffer; return byteBuffer;
} }
public long getBlockOffset() { public long getBlockOffset() {
return blockOffset; return blockOffset;
} }
private void writeBufferToByteBuffer() { private void writeBufferToByteBuffer() {
byteBuffer.position(0); byteBuffer.position(0);
byteBuffer.put(buffer); byteBuffer.put(buffer);
} }
public void writeAsync() { public void writeAsync() {
writeBufferToByteBuffer(); writeBufferToByteBuffer();
} }
public void force() { public void force() {
// some tests use HeapByteBuffer and don't support force // some tests use HeapByteBuffer and don't support force
if (byteBuffer instanceof MappedByteBuffer) { if (byteBuffer instanceof MappedByteBuffer) {
((MappedByteBuffer) byteBuffer).force(); ((MappedByteBuffer) byteBuffer).force();
} }
} }
@Override @Override
public String toString() { public String toString() {
return "DiskBlock[" + blockOffset + "]"; return "DiskBlock[" + blockOffset + "]";
} }
} }

438
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorage.java
View File

@@ -14,273 +14,273 @@ import org.slf4j.LoggerFactory;
public class DiskStorage implements AutoCloseable { public class DiskStorage implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class); private static final Logger LOGGER = LoggerFactory.getLogger(DiskStorage.class);
private static final long FREE_LIST_ROOT_OFFSET = 0; private static final long FREE_LIST_ROOT_OFFSET = 0;
private static final long NO_POINTER = 0; private static final long NO_POINTER = 0;
private static final int FREE_LIST_NEXT_POINTER = 0; private static final int FREE_LIST_NEXT_POINTER = 0;
private static final int FREE_LIST_PREV_POINTER = 8; private static final int FREE_LIST_PREV_POINTER = 8;
private static final int FREE_LIST_SIZE = 16; private static final int FREE_LIST_SIZE = 16;
private static final int FREE_LIST_NODE_SIZE = 32; private static final int FREE_LIST_NODE_SIZE = 32;
private final FileChannel fileChannel; private final FileChannel fileChannel;
private Path relativeDatabaseFileForLogging; private Path relativeDatabaseFileForLogging;
public DiskStorage(final Path databaseFile, Path storageBasePath) {
public DiskStorage(final Path databaseFile, Path storageBasePath) { this.relativeDatabaseFileForLogging = storageBasePath != null ? storageBasePath.relativize(databaseFile)
this.relativeDatabaseFileForLogging = storageBasePath != null ? storageBasePath.relativize(databaseFile): databaseFile; : databaseFile;
try { try {
Files.createDirectories(databaseFile.getParent()); Files.createDirectories(databaseFile.getParent());
fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE, fileChannel = FileChannel.open(databaseFile, StandardOpenOption.READ, StandardOpenOption.WRITE,
StandardOpenOption.CREATE); StandardOpenOption.CREATE);
initIfNew(); initIfNew();
} catch (final IOException e) { } catch (final IOException e) {
throw new DiskStorageException(e); throw new DiskStorageException(e);
} }
} }
private void initIfNew() throws IOException { private void initIfNew() throws IOException {
if (fileChannel.size() == 0) { if (fileChannel.size() == 0) {
// file is new -> add root of the free list // file is new -> add root of the free list
writeFreeListRootNodePosition(NO_POINTER); writeFreeListRootNodePosition(NO_POINTER);
} }
} }
public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) { public DiskBlock getDiskBlock(final long blockOffset, final int blockSize) {
try { try {
LOGGER.trace("read block={} file={}", blockOffset, relativeDatabaseFileForLogging); LOGGER.trace("read block={} file={}", blockOffset, relativeDatabaseFileForLogging);
final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize); final var byteBuffer = fileChannel.map(MapMode.READ_WRITE, blockOffset, blockSize);
return new DiskBlock(blockOffset, byteBuffer); return new DiskBlock(blockOffset, byteBuffer);
} catch (final IOException e) { } catch (final IOException e) {
throw new DiskStorageException(e); throw new DiskStorageException(e);
} }
} }
public Path getRelativeDatabaseFileForLogging() {
return relativeDatabaseFileForLogging;
}
@Override public Path getRelativeDatabaseFileForLogging() {
public void close() { return relativeDatabaseFileForLogging;
try { }
fileChannel.force(true);
fileChannel.close();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public synchronized long allocateBlock(final int blockSize) { @Override
if (blockSize < FREE_LIST_NODE_SIZE) { public void close() {
throw new IllegalArgumentException("The minimal allocation size is 32 byte."); try {
} fileChannel.force(true);
fileChannel.close();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
try { public synchronized long allocateBlock(final int blockSize) {
final var optionalFreeBlock = findFreeBlockWithSize(blockSize); if (blockSize < FREE_LIST_NODE_SIZE) {
if (optionalFreeBlock.isPresent()) { throw new IllegalArgumentException("The minimal allocation size is 32 byte.");
final FreeListNode freeBlock = optionalFreeBlock.get(); }
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
private long allocateNewBlock(final int blockSize) throws IOException { try {
final var buffer = new byte[blockSize]; final var optionalFreeBlock = findFreeBlockWithSize(blockSize);
final var src = ByteBuffer.wrap(buffer); if (optionalFreeBlock.isPresent()) {
final FreeListNode freeBlock = optionalFreeBlock.get();
removeBlockFromFreeList(freeBlock);
clearBlock(freeBlock);
return freeBlock.getOffset();
} else {
return allocateNewBlock(blockSize);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
// block numbers start with 1, so that the uninitialized value private long allocateNewBlock(final int blockSize) throws IOException {
// (0) means 'no block'. That way we do not have to write final var buffer = new byte[blockSize];
// data to a newly created block, which reduces IO. final var src = ByteBuffer.wrap(buffer);
final var blockOffset = fileChannel.size();
fileChannel.write(src, fileChannel.size());
return blockOffset;
}
public synchronized void free(final long blockOffset, final int blockSize) throws IOException { // block numbers start with 1, so that the uninitialized value
// (0) means 'no block'. That way we do not have to write
// data to a newly created block, which reduces IO.
final var blockOffset = fileChannel.size();
fileChannel.write(src, fileChannel.size());
return blockOffset;
}
final var neighboringFreeListNode = getNeighboringFreeListNode(blockOffset); public synchronized void free(final long blockOffset, final int blockSize) throws IOException {
if (neighboringFreeListNode.isPresent()) { final var neighboringFreeListNode = getNeighboringFreeListNode(blockOffset);
// insert new free node into the free list
final var prev = neighboringFreeListNode.get();
insertFreeListNode(prev, blockOffset, blockSize); if (neighboringFreeListNode.isPresent()) {
// insert new free node into the free list
final var prev = neighboringFreeListNode.get();
} else { insertFreeListNode(prev, blockOffset, blockSize);
// add new free list node as the first node in the list
insertFreeListNodeAsNewRoot(blockOffset, blockSize);
}
}
private void insertFreeListNodeAsNewRoot(final long blockOffset, final int blockSize) throws IOException { } else {
final var freeListRootNodePosition = readFreeListRootNodePosition(); // add new free list node as the first node in the list
insertFreeListNodeAsNewRoot(blockOffset, blockSize);
}
}
if (freeListRootNodePosition > 0) { private void insertFreeListNodeAsNewRoot(final long blockOffset, final int blockSize) throws IOException {
// there are free list nodes, but they are after the new node final var freeListRootNodePosition = readFreeListRootNodePosition();
final var next = readFreeListNode(freeListRootNodePosition); if (freeListRootNodePosition > 0) {
final var newNode = new FreeListNode(blockOffset, blockSize); // there are free list nodes, but they are after the new node
FreeListNode.link(newNode, next); final var next = readFreeListNode(freeListRootNodePosition);
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode, next); FreeListNode.link(newNode, next);
writeFreeListRootNodePosition(blockOffset);
} else { writeFreeListNode(newNode, next);
// this is the first free list node writeFreeListRootNodePosition(blockOffset);
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode);
writeFreeListRootNodePosition(blockOffset);
}
}
private void insertFreeListNode(final FreeListNode prev, final long blockOffset, final int blockSize) } else {
throws IOException { // this is the first free list node
final var newNode = new FreeListNode(blockOffset, blockSize);
writeFreeListNode(newNode);
writeFreeListRootNodePosition(blockOffset);
}
}
final var newNode = new FreeListNode(blockOffset, blockSize); private void insertFreeListNode(final FreeListNode prev, final long blockOffset, final int blockSize)
final var next = prev.hasNext() ? readFreeListNode(prev.getNext()) : null; throws IOException {
FreeListNode.link(prev, newNode, next); final var newNode = new FreeListNode(blockOffset, blockSize);
final var next = prev.hasNext() ? readFreeListNode(prev.getNext()) : null;
writeFreeListNode(prev, newNode, next); FreeListNode.link(prev, newNode, next);
}
/** writeFreeListNode(prev, newNode, next);
* }
* @param blockOffset the offset of the block that is about to be free'd
* @return the free list node before the block
* @throws IOException
*/
private Optional<FreeListNode> getNeighboringFreeListNode(final long blockOffset) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListRootNodePosition < blockOffset) {
long nextFreeListNodeOffset = freeListRootNodePosition; /**
while (nextFreeListNodeOffset > 0) { *
final var freeListNode = readFreeListNode(nextFreeListNodeOffset); * @param blockOffset the offset of the block that is about to be free'd
* @return the free list node before the block
* @throws IOException
*/
private Optional<FreeListNode> getNeighboringFreeListNode(final long blockOffset) throws IOException {
FreeListNode result = null;
final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListRootNodePosition < blockOffset) {
if (freeListNode.getOffset() > blockOffset) { long nextFreeListNodeOffset = freeListRootNodePosition;
break; while (nextFreeListNodeOffset > 0) {
} final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
nextFreeListNodeOffset = freeListNode.getNext();
result = freeListNode;
}
}
return Optional.ofNullable(result);
}
private Optional<FreeListNode> findFreeBlockWithSize(final long blockSize) throws IOException { if (freeListNode.getOffset() > blockOffset) {
FreeListNode result = null; break;
final long freeListRootNodePosition = readFreeListRootNodePosition(); }
nextFreeListNodeOffset = freeListNode.getNext();
result = freeListNode;
}
}
return Optional.ofNullable(result);
}
long nextFreeListNodeOffset = freeListRootNodePosition; private Optional<FreeListNode> findFreeBlockWithSize(final long blockSize) throws IOException {
while (nextFreeListNodeOffset > 0) { FreeListNode result = null;
final var freeListNode = readFreeListNode(nextFreeListNodeOffset); final long freeListRootNodePosition = readFreeListRootNodePosition();
if (freeListNode.getSize() == blockSize) { long nextFreeListNodeOffset = freeListRootNodePosition;
result = freeListNode; while (nextFreeListNodeOffset > 0) {
break; final var freeListNode = readFreeListNode(nextFreeListNodeOffset);
}
nextFreeListNodeOffset = freeListNode.getNext();
}
return Optional.ofNullable(result); if (freeListNode.getSize() == blockSize) {
} result = freeListNode;
break;
}
nextFreeListNodeOffset = freeListNode.getNext();
}
private void clearBlock(final FreeListNode freeBlock) throws IOException { return Optional.ofNullable(result);
final var src = ByteBuffer.allocate(freeBlock.getSize()); }
fileChannel.write(src, freeBlock.getOffset());
}
private void removeBlockFromFreeList(final FreeListNode freeBlock) throws IOException { private void clearBlock(final FreeListNode freeBlock) throws IOException {
final var src = ByteBuffer.allocate(freeBlock.getSize());
fileChannel.write(src, freeBlock.getOffset());
}
if (freeBlock.getPrev() == 0) { private void removeBlockFromFreeList(final FreeListNode freeBlock) throws IOException {
writeFreeListRootNodePosition(freeBlock.getNext());
}
if (freeBlock.getNext() > 0) { if (freeBlock.getPrev() == 0) {
final FreeListNode next = readFreeListNode(freeBlock.getNext()); writeFreeListRootNodePosition(freeBlock.getNext());
next.setPrev(freeBlock.getPrev()); }
writeFreeListNode(next);
}
if (freeBlock.getPrev() > 0) { if (freeBlock.getNext() > 0) {
final FreeListNode prev = readFreeListNode(freeBlock.getPrev()); final FreeListNode next = readFreeListNode(freeBlock.getNext());
prev.setNext(freeBlock.getNext()); next.setPrev(freeBlock.getPrev());
writeFreeListNode(prev); writeFreeListNode(next);
} }
}
private FreeListNode readFreeListNode(final long freeListNodePosition) throws IOException { if (freeBlock.getPrev() > 0) {
final var freeListNode = ByteBuffer.allocate(FREE_LIST_NODE_SIZE); final FreeListNode prev = readFreeListNode(freeBlock.getPrev());
fileChannel.read(freeListNode, freeListNodePosition); prev.setNext(freeBlock.getNext());
final long offset = freeListNodePosition; writeFreeListNode(prev);
final long next = freeListNode.getLong(FREE_LIST_NEXT_POINTER); }
final long prev = freeListNode.getLong(FREE_LIST_PREV_POINTER); }
final int size = freeListNode.getInt(FREE_LIST_SIZE);
return new FreeListNode(offset, next, prev, size);
}
private void writeFreeListNode(final FreeListNode... nodes) throws IOException { private FreeListNode readFreeListNode(final long freeListNodePosition) throws IOException {
final var freeListNode = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
fileChannel.read(freeListNode, freeListNodePosition);
final long offset = freeListNodePosition;
final long next = freeListNode.getLong(FREE_LIST_NEXT_POINTER);
final long prev = freeListNode.getLong(FREE_LIST_PREV_POINTER);
final int size = freeListNode.getInt(FREE_LIST_SIZE);
return new FreeListNode(offset, next, prev, size);
}
for (final FreeListNode node : nodes) { private void writeFreeListNode(final FreeListNode... nodes) throws IOException {
if (node != null) {
final var src = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
src.putLong(FREE_LIST_NEXT_POINTER, node.getNext());
src.putLong(FREE_LIST_PREV_POINTER, node.getPrev());
src.putInt(FREE_LIST_SIZE, node.getSize());
fileChannel.write(src, node.getOffset());
}
}
}
private long readFreeListRootNodePosition() throws IOException { for (final FreeListNode node : nodes) {
final var freeListFirstBlock = ByteBuffer.allocate(8); if (node != null) {
fileChannel.read(freeListFirstBlock, FREE_LIST_ROOT_OFFSET); final var src = ByteBuffer.allocate(FREE_LIST_NODE_SIZE);
return freeListFirstBlock.getLong(0); src.putLong(FREE_LIST_NEXT_POINTER, node.getNext());
} src.putLong(FREE_LIST_PREV_POINTER, node.getPrev());
src.putInt(FREE_LIST_SIZE, node.getSize());
fileChannel.write(src, node.getOffset());
}
}
}
private void writeFreeListRootNodePosition(final long freeListRootNodePosition) throws IOException { private long readFreeListRootNodePosition() throws IOException {
final var freeListFirstBlock = ByteBuffer.allocate(8); final var freeListFirstBlock = ByteBuffer.allocate(8);
freeListFirstBlock.putLong(0, freeListRootNodePosition); fileChannel.read(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
fileChannel.write(freeListFirstBlock, FREE_LIST_ROOT_OFFSET); return freeListFirstBlock.getLong(0);
} }
public synchronized void ensureAlignmentForNewBlocks(final int alignment) { private void writeFreeListRootNodePosition(final long freeListRootNodePosition) throws IOException {
try { final var freeListFirstBlock = ByteBuffer.allocate(8);
final long size = fileChannel.size(); freeListFirstBlock.putLong(0, freeListRootNodePosition);
final int alignmentMismatch = Math.floorMod(size, alignment); fileChannel.write(freeListFirstBlock, FREE_LIST_ROOT_OFFSET);
if (alignmentMismatch != 0) { }
// The next allocated block would not be aligned. Therefore we allocate a
// throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public long size() { public synchronized void ensureAlignmentForNewBlocks(final int alignment) {
try { try {
return fileChannel.size(); final long size = fileChannel.size();
} catch (final IOException e) { final int alignmentMismatch = Math.floorMod(size, alignment);
throw new DiskStorageException(e); if (alignmentMismatch != 0) {
} // The next allocated block would not be aligned. Therefore we allocate a
} // throw-away block.
allocateNewBlock(alignment - alignmentMismatch);
}
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public int minAllocationSize() { public long size() {
return FREE_LIST_NODE_SIZE; try {
} return fileChannel.size();
} catch (final IOException e) {
throw new DiskStorageException(e);
}
}
public int minAllocationSize() {
return FREE_LIST_NODE_SIZE;
}
} }

20
block-storage/src/main/java/org/lucares/pdb/diskstorage/DiskStorageException.java
View File

@@ -2,18 +2,18 @@ package org.lucares.pdb.diskstorage;
public class DiskStorageException extends RuntimeException { public class DiskStorageException extends RuntimeException {
private static final long serialVersionUID = 1683775743640383633L; private static final long serialVersionUID = 1683775743640383633L;
public DiskStorageException(final String message, final Throwable cause) { public DiskStorageException(final String message, final Throwable cause) {
super(message, cause); super(message, cause);
} }
public DiskStorageException(final String message) { public DiskStorageException(final String message) {
super(message); super(message);
} }
public DiskStorageException(final Throwable cause) { public DiskStorageException(final Throwable cause) {
super(cause); super(cause);
} }
} }

124
block-storage/src/main/java/org/lucares/pdb/diskstorage/FreeListNode.java
View File

@@ -1,82 +1,82 @@
package org.lucares.pdb.diskstorage; package org.lucares.pdb.diskstorage;
public class FreeListNode { public class FreeListNode {
private final long offset; private final long offset;
private long next; private long next;
private long prev; private long prev;
private int size; private int size;
public FreeListNode(final long offset, final int size) { public FreeListNode(final long offset, final int size) {
this.offset = offset; this.offset = offset;
this.size = size; this.size = size;
} }
public FreeListNode(final long offset, final long next, final long prev, final int size) { public FreeListNode(final long offset, final long next, final long prev, final int size) {
this.offset = offset; this.offset = offset;
this.next = next; this.next = next;
this.prev = prev; this.prev = prev;
this.size = size; this.size = size;
} }
public long getOffset() { public long getOffset() {
return offset; return offset;
} }
public long getNext() { public long getNext() {
return next; return next;
} }
public void setNext(final long next) { public void setNext(final long next) {
this.next = next; this.next = next;
} }
public void setNext(final FreeListNode next) { public void setNext(final FreeListNode next) {
this.next = next != null ? next.getOffset() : 0; this.next = next != null ? next.getOffset() : 0;
} }
public long getPrev() { public long getPrev() {
return prev; return prev;
} }
public void setPrev(final long prev) { public void setPrev(final long prev) {
this.prev = prev; this.prev = prev;
} }
public void setPrev(final FreeListNode prev) { public void setPrev(final FreeListNode prev) {
this.prev = prev != null ? prev.getOffset() : 0; this.prev = prev != null ? prev.getOffset() : 0;
} }
public int getSize() { public int getSize() {
return size; return size;
} }
public void setSize(final int size) { public void setSize(final int size) {
this.size = size; this.size = size;
} }
@Override @Override
public String toString() { public String toString() {
return "FreeListNode [offset=" + offset + ", next=" + next + ", prev=" + prev + ", size=" + size + "]"; return "FreeListNode [offset=" + offset + ", next=" + next + ", prev=" + prev + ", size=" + size + "]";
} }
public boolean hasNext() { public boolean hasNext() {
return next != 0; return next != 0;
} }
public static void link(final FreeListNode prev, final FreeListNode next) { public static void link(final FreeListNode prev, final FreeListNode next) {
prev.setNext(next); prev.setNext(next);
next.setPrev(prev); next.setPrev(prev);
} }
public static void link(final FreeListNode prev, final FreeListNode middle, final FreeListNode next) { public static void link(final FreeListNode prev, final FreeListNode middle, final FreeListNode next) {
if (prev != null) { if (prev != null) {
prev.setNext(middle); prev.setNext(middle);
} }
middle.setPrev(prev); middle.setPrev(prev);
middle.setNext(next); middle.setNext(next);
if (next != null) { if (next != null) {
next.setPrev(prev); next.setPrev(prev);
} }
} }
} }

122
block-storage/src/main/java/org/lucares/pdb/map/ByteArrayKey.java
View File

@@ -3,77 +3,77 @@ package org.lucares.pdb.map;
import java.util.Arrays; import java.util.Arrays;
public final class ByteArrayKey implements Comparable<ByteArrayKey> { public final class ByteArrayKey implements Comparable<ByteArrayKey> {
private final byte[] bytes; private final byte[] bytes;
public ByteArrayKey(final byte[] bytes) { public ByteArrayKey(final byte[] bytes) {
this.bytes = bytes; this.bytes = bytes;
} }
@Override @Override
public int compareTo(final ByteArrayKey o) { public int compareTo(final ByteArrayKey o) {
return compare(bytes, o.bytes); return compare(bytes, o.bytes);
} }
public static int compare(final byte[] key, final byte[] otherKey) { public static int compare(final byte[] key, final byte[] otherKey) {
return Arrays.compare(key, otherKey); return Arrays.compare(key, otherKey);
} }
public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) { public static boolean isPrefix(final byte[] key, final byte[] keyPrefix) {
return compareKeyPrefix(key, keyPrefix) == 0; return compareKeyPrefix(key, keyPrefix) == 0;
} }
/** /**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the * Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key * key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix. * is longer than the prefix.
* *
* @param prefix the prefix * @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is * @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])} * defined by {@link #compare(byte[])}
*/ */
public static int compareKeyPrefix(final byte[] key, final byte[] prefix) { public static int compareKeyPrefix(final byte[] key, final byte[] prefix) {
int i = 0; int i = 0;
while (i < key.length && i < prefix.length) { while (i < key.length && i < prefix.length) {
if (key[i] != prefix[i]) { if (key[i] != prefix[i]) {
return key[i] - prefix[i]; return key[i] - prefix[i];
} }
i++; i++;
} }
return key.length > prefix.length ? 0 : key.length - prefix.length; return key.length > prefix.length ? 0 : key.length - prefix.length;
} }
public static boolean equal(final byte[] key, final byte[] otherKey) { public static boolean equal(final byte[] key, final byte[] otherKey) {
return compare(key, otherKey) == 0; return compare(key, otherKey) == 0;
} }
@Override
public String toString() {
return Arrays.toString(bytes);
}
@Override @Override
public int hashCode() { public String toString() {
final int prime = 31; return Arrays.toString(bytes);
int result = 1; }
result = prime * result + Arrays.hashCode(bytes);
return result;
}
@Override @Override
public boolean equals(final Object obj) { public int hashCode() {
if (this == obj) final int prime = 31;
return true; int result = 1;
if (obj == null) result = prime * result + Arrays.hashCode(bytes);
return false; return result;
if (getClass() != obj.getClass()) }
return false;
final ByteArrayKey other = (ByteArrayKey) obj; @Override
if (!Arrays.equals(bytes, other.bytes)) public boolean equals(final Object obj) {
return false; if (this == obj)
return true; return true;
} if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ByteArrayKey other = (ByteArrayKey) obj;
if (!Arrays.equals(bytes, other.bytes))
return false;
return true;
}
} }

14
block-storage/src/main/java/org/lucares/pdb/map/Empty.java
View File

@@ -14,13 +14,13 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
* {@link Empty} solves this by providing a single unmodifiable value. * {@link Empty} solves this by providing a single unmodifiable value.
*/ */
public final class Empty { public final class Empty {
public static final Empty INSTANCE = new Empty(); public static final Empty INSTANCE = new Empty();
private Empty() { private Empty() {
} }
@Override @Override
public String toString() { public String toString() {
return "<empty>"; return "<empty>";
} }
} }

244
block-storage/src/main/java/org/lucares/pdb/map/NodeEntry.java
View File

@@ -9,158 +9,158 @@ import java.util.function.Predicate;
import org.lucares.utils.byteencoder.VariableByteEncoder; import org.lucares.utils.byteencoder.VariableByteEncoder;
class NodeEntry { class NodeEntry {
enum ValueType { enum ValueType {
VALUE_INLINE((byte) 1), NODE_POINTER((byte) 2); VALUE_INLINE((byte) 1), NODE_POINTER((byte) 2);
private final byte b; private final byte b;
ValueType(final byte b) { ValueType(final byte b) {
this.b = b; this.b = b;
} }
static ValueType fromByte(final byte b) { static ValueType fromByte(final byte b) {
for (final ValueType type : values()) { for (final ValueType type : values()) {
if (type.b == b) { if (type.b == b) {
return type; return type;
} }
} }
throw new IllegalStateException("Cannot map byte " + b + " to a value type."); throw new IllegalStateException("Cannot map byte " + b + " to a value type.");
} }
public byte asByte() { public byte asByte() {
return b; return b;
} }
} }
static final class KeyMatches implements Predicate<NodeEntry> { static final class KeyMatches implements Predicate<NodeEntry> {
private final byte[] key; private final byte[] key;
public KeyMatches(final byte[] key) { public KeyMatches(final byte[] key) {
this.key = key; this.key = key;
} }
@Override @Override
public boolean test(final NodeEntry t) { public boolean test(final NodeEntry t) {
return Arrays.equals(key, t.getKey()); return Arrays.equals(key, t.getKey());
} }
} }
private final ValueType type; private final ValueType type;
private final byte[] key; private final byte[] key;
private final byte[] value; private final byte[] value;
public NodeEntry(final ValueType type, final byte[] key, final byte[] value) { public NodeEntry(final ValueType type, final byte[] key, final byte[] value) {
this.type = type; this.type = type;
this.key = key; this.key = key;
this.value = value; this.value = value;
} }
public ValueType getType() { public ValueType getType() {
return type; return type;
} }
public byte[] getKey() { public byte[] getKey() {
return key; return key;
} }
public byte[] getValue() { public byte[] getValue() {
return value; return value;
} }
public int size() { public int size() {
return 1 + key.length + value.length; return 1 + key.length + value.length;
} }
@Override @Override
public String toString() { public String toString() {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value)) final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: new String(value, StandardCharsets.UTF_8); : new String(value, StandardCharsets.UTF_8);
return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value=" return "NodeEntry [type=" + type + ", key=" + new String(key, StandardCharsets.UTF_8) + ", value="
+ valueAsString + "]"; + valueAsString + "]";
} }
public <K,V> String toString(final Function<byte[], K> keyDecoder, final Function<byte[], V> valueDecoder) { public <K, V> String toString(final Function<byte[], K> keyDecoder, final Function<byte[], V> valueDecoder) {
final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value)) final String valueAsString = isInnerNode() ? String.valueOf(VariableByteEncoder.decodeFirstValue(value))
: String.valueOf(valueDecoder.apply(value)); : String.valueOf(valueDecoder.apply(value));
final String keyAsString; final String keyAsString;
if (Arrays.equals(key, PersistentMap.MAX_KEY)) { if (Arrays.equals(key, PersistentMap.MAX_KEY)) {
keyAsString = "<<<MAX_KEY>>>"; keyAsString = "<<<MAX_KEY>>>";
} else { } else {
keyAsString = String.valueOf(keyDecoder.apply(key)); keyAsString = String.valueOf(keyDecoder.apply(key));
} }
return "NodeEntry [type=" + type + ", key=" + keyAsString + ", value=" + valueAsString + "]"; return "NodeEntry [type=" + type + ", key=" + keyAsString + ", value=" + valueAsString + "]";
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + Arrays.hashCode(key); result = prime * result + Arrays.hashCode(key);
result = prime * result + ((type == null) ? 0 : type.hashCode()); result = prime * result + ((type == null) ? 0 : type.hashCode());
result = prime * result + Arrays.hashCode(value); result = prime * result + Arrays.hashCode(value);
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final NodeEntry other = (NodeEntry) obj; final NodeEntry other = (NodeEntry) obj;
if (!Arrays.equals(key, other.key)) if (!Arrays.equals(key, other.key))
return false; return false;
if (type != other.type) if (type != other.type)
return false; return false;
if (!Arrays.equals(value, other.value)) if (!Arrays.equals(value, other.value))
return false; return false;
return true; return true;
} }
public static int neededBytes(final Collection<NodeEntry> entries) { public static int neededBytes(final Collection<NodeEntry> entries) {
return entries.stream().mapToInt(NodeEntry::size).sum(); return entries.stream().mapToInt(NodeEntry::size).sum();
} }
public int compare(final byte[] otherKey) { public int compare(final byte[] otherKey) {
return ByteArrayKey.compare(key, otherKey); return ByteArrayKey.compare(key, otherKey);
} }
public boolean isPrefix(final byte[] keyPrefix) { public boolean isPrefix(final byte[] keyPrefix) {
return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0; return ByteArrayKey.compareKeyPrefix(key, keyPrefix) == 0;
} }
/** /**
* Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the * Same as {@link #compare(byte[])}, but return 0 if prefix is a prefix of the
* key. {@link #compare(byte[])} return values &gt;0 in that case, because key * key. {@link #compare(byte[])} return values &gt;0 in that case, because key
* is longer than the prefix. * is longer than the prefix.
* *
* @param prefix the prefix * @param prefix the prefix
* @return 0 if {@code prefix} is a prefix of the key otherwise the value is * @return 0 if {@code prefix} is a prefix of the key otherwise the value is
* defined by {@link #compare(byte[])} * defined by {@link #compare(byte[])}
*/ */
public int compareKeyPrefix(final byte[] prefix) { public int compareKeyPrefix(final byte[] prefix) {
return ByteArrayKey.compareKeyPrefix(key, prefix); return ByteArrayKey.compareKeyPrefix(key, prefix);
} }
public boolean equal(final byte[] otherKey) { public boolean equal(final byte[] otherKey) {
return compare(otherKey) == 0; return compare(otherKey) == 0;
} }
public boolean isDataNode() { public boolean isDataNode() {
return type == ValueType.VALUE_INLINE; return type == ValueType.VALUE_INLINE;
} }
public boolean isInnerNode() { public boolean isInnerNode() {
return type == ValueType.NODE_POINTER; return type == ValueType.NODE_POINTER;
} }
} }

884
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -23,470 +23,470 @@ import org.slf4j.LoggerFactory;
public class PersistentMap<K, V> implements AutoCloseable { public class PersistentMap<K, V> implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class); private static final Logger LOGGER = LoggerFactory.getLogger(PersistentMap.class);
// the maximum key // the maximum key
static final byte[] MAX_KEY; static final byte[] MAX_KEY;
static { static {
MAX_KEY = new byte[20]; MAX_KEY = new byte[20];
Arrays.fill(MAX_KEY, Byte.MAX_VALUE); Arrays.fill(MAX_KEY, Byte.MAX_VALUE);
} }
interface VisitorCallback { interface VisitorCallback {
void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth); void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth);
} }
public interface EncoderDecoder<O> { public interface EncoderDecoder<O> {
public byte[] encode(O object); public byte[] encode(O object);
public O decode(byte[] bytes); public O decode(byte[] bytes);
public default Function<byte[], O> asDecoder() { public default Function<byte[], O> asDecoder() {
return bytes -> this.decode(bytes); return bytes -> this.decode(bytes);
} }
public default Function<O, byte[]> asEncoder() { public default Function<O, byte[]> asEncoder() {
return plain -> this.encode(plain); return plain -> this.encode(plain);
} }
public byte[] getEmptyValue(); public byte[] getEmptyValue();
} }
private static final class StringCoder implements EncoderDecoder<String> { private static final class StringCoder implements EncoderDecoder<String> {
@Override @Override
public byte[] encode(final String object) { public byte[] encode(final String object) {
return object.getBytes(StandardCharsets.UTF_8); return object.getBytes(StandardCharsets.UTF_8);
} }
@Override @Override
public String decode(final byte[] bytes) { public String decode(final byte[] bytes) {
return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8); return bytes == null ? null : new String(bytes, StandardCharsets.UTF_8);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0 }; return new byte[] { 0 };
} }
} }
private static final class LongCoder implements EncoderDecoder<Long> { private static final class LongCoder implements EncoderDecoder<Long> {
@Override @Override
public byte[] encode(final Long object) { public byte[] encode(final Long object) {
return VariableByteEncoder.encode(object); return VariableByteEncoder.encode(object);
} }
@Override @Override
public Long decode(final byte[] bytes) { public Long decode(final byte[] bytes) {
return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes); return bytes == null ? null : VariableByteEncoder.decodeFirstValue(bytes);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0 }; return new byte[] { 0 };
} }
} }
private static final class UUIDCoder implements EncoderDecoder<UUID> { private static final class UUIDCoder implements EncoderDecoder<UUID> {
@Override @Override
public byte[] encode(final UUID uuid) { public byte[] encode(final UUID uuid) {
final long mostSignificantBits = uuid.getMostSignificantBits(); final long mostSignificantBits = uuid.getMostSignificantBits();
final long leastSignificantBits = uuid.getLeastSignificantBits(); final long leastSignificantBits = uuid.getLeastSignificantBits();
return VariableByteEncoder.encode(mostSignificantBits, leastSignificantBits); return VariableByteEncoder.encode(mostSignificantBits, leastSignificantBits);
} }
@Override @Override
public UUID decode(final byte[] bytes) { public UUID decode(final byte[] bytes) {
final LongList longs = VariableByteEncoder.decode(bytes); final LongList longs = VariableByteEncoder.decode(bytes);
final long mostSignificantBits = longs.get(0); final long mostSignificantBits = longs.get(0);
final long leastSignificantBits = longs.get(1); final long leastSignificantBits = longs.get(1);
return new UUID(mostSignificantBits, leastSignificantBits); return new UUID(mostSignificantBits, leastSignificantBits);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0 }; return new byte[] { 0 };
} }
} }
private static final class EmptyCoder implements EncoderDecoder<Empty> { private static final class EmptyCoder implements EncoderDecoder<Empty> {
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
@Override @Override
public byte[] encode(final Empty __) { public byte[] encode(final Empty __) {
return EMPTY_BYTE_ARRAY; return EMPTY_BYTE_ARRAY;
} }
@Override @Override
public Empty decode(final byte[] bytes) { public Empty decode(final byte[] bytes) {
Preconditions.checkTrue(bytes.length == 0, ""); Preconditions.checkTrue(bytes.length == 0, "");
return Empty.INSTANCE; return Empty.INSTANCE;
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] {}; return new byte[] {};
} }
} }
public static final EncoderDecoder<Long> LONG_CODER = new LongCoder(); public static final EncoderDecoder<Long> LONG_CODER = new LongCoder();
public static final EncoderDecoder<UUID> UUID_ENCODER = new UUIDCoder(); public static final EncoderDecoder<UUID> UUID_ENCODER = new UUIDCoder();
public static final EncoderDecoder<String> STRING_CODER = new StringCoder(); public static final EncoderDecoder<String> STRING_CODER = new StringCoder();
public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder(); public static final EncoderDecoder<Empty> EMPTY_ENCODER = new EmptyCoder();
static final int BLOCK_SIZE = 4096; static final int BLOCK_SIZE = 4096;
static final long NODE_OFFSET_TO_ROOT_NODE = 8; static final long NODE_OFFSET_TO_ROOT_NODE = 8;
private final DiskStorage diskStore; private final DiskStorage diskStore;
private int maxEntriesInNode = Integer.MAX_VALUE; private int maxEntriesInNode = Integer.MAX_VALUE;
private final EncoderDecoder<K> keyEncoder; private final EncoderDecoder<K> keyEncoder;
private final EncoderDecoder<V> valueEncoder; private final EncoderDecoder<V> valueEncoder;
private final LRUCache<Long, PersistentMapDiskNode> nodeCache = new LRUCache<>(10_000); private final LRUCache<Long, PersistentMapDiskNode> nodeCache = new LRUCache<>(10_000);
private final LRUCache<K, V> valueCache = new LRUCache<>(1_000); private final LRUCache<K, V> valueCache = new LRUCache<>(1_000);
public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder, public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder,
final EncoderDecoder<V> valueEncoder) { final EncoderDecoder<V> valueEncoder) {
this.diskStore = new DiskStorage(path, storageBasePath); this.diskStore = new DiskStorage(path, storageBasePath);
this.keyEncoder = keyEncoder; this.keyEncoder = keyEncoder;
this.valueEncoder = valueEncoder; this.valueEncoder = valueEncoder;
initIfNew(); initIfNew();
} }
@Override @Override
public void close() { public void close() {
diskStore.close(); diskStore.close();
} }
public void setMaxEntriesInNode(final int maxEntriesInNode) { public void setMaxEntriesInNode(final int maxEntriesInNode) {
this.maxEntriesInNode = maxEntriesInNode; this.maxEntriesInNode = maxEntriesInNode;
} }
private void initIfNew() { private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) { if (diskStore.size() < BLOCK_SIZE) {
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize()); final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE, Preconditions.checkEqual(nodeOffsetToRootNode, NODE_OFFSET_TO_ROOT_NODE,
"The offset of the pointer to the root node must be at a well known location. " "The offset of the pointer to the root node must be at a well known location. "
+ "Otherwise we would not be able to find it in an already existing file."); + "Otherwise we would not be able to find it in an already existing file.");
// 2. make sure new blocks are aligned to the block size (for faster disk IO) // 2. make sure new blocks are aligned to the block size (for faster disk IO)
diskStore.ensureAlignmentForNewBlocks(BLOCK_SIZE); diskStore.ensureAlignmentForNewBlocks(BLOCK_SIZE);
// 3. initialize an empty root node // 3. initialize an empty root node
final long blockOffset = diskStore.allocateBlock(BLOCK_SIZE); final long blockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset); final var rootNode = PersistentMapDiskNode.emptyRootNode(blockOffset);
writeNode(rootNode); writeNode(rootNode);
// 4. update pointer to root node // 4. update pointer to root node
writeNodeOffsetOfRootNode(blockOffset); writeNodeOffsetOfRootNode(blockOffset);
// 5. insert a dummy entry with a 'maximum' key // 5. insert a dummy entry with a 'maximum' key
putValue(MAX_KEY, valueEncoder.getEmptyValue()); putValue(MAX_KEY, valueEncoder.getEmptyValue());
} }
} }
public synchronized void putAllValues(final Map<K, V> map) { public synchronized void putAllValues(final Map<K, V> map) {
for (final Entry<K, V> e : map.entrySet()) { for (final Entry<K, V> e : map.entrySet()) {
putValue(e.getKey(), e.getValue()); putValue(e.getKey(), e.getValue());
} }
} }
public synchronized V putValue(final K key, final V value) { public synchronized V putValue(final K key, final V value) {
final V cachedValue = valueCache.get(key); final V cachedValue = valueCache.get(key);
if (cachedValue != null && cachedValue == value) { if (cachedValue != null && cachedValue == value) {
return value; return value;
} }
final byte[] encodedKey = keyEncoder.encode(key); final byte[] encodedKey = keyEncoder.encode(key);
final byte[] encodedValue = valueEncoder.encode(value); final byte[] encodedValue = valueEncoder.encode(value);
final byte[] encodedOldValue = putValue(encodedKey, encodedValue); final byte[] encodedOldValue = putValue(encodedKey, encodedValue);
final V oldValue = encodedOldValue == null ? null : valueEncoder.decode(encodedOldValue); final V oldValue = encodedOldValue == null ? null : valueEncoder.decode(encodedOldValue);
valueCache.put(key, value); valueCache.put(key, value);
return oldValue; return oldValue;
} }
public synchronized V getValue(final K key) { public synchronized V getValue(final K key) {
final V cachedValue = valueCache.get(key); final V cachedValue = valueCache.get(key);
if (cachedValue != null) { if (cachedValue != null) {
return cachedValue; return cachedValue;
} }
final byte[] encodedKey = keyEncoder.encode(key); final byte[] encodedKey = keyEncoder.encode(key);
final byte[] foundValue = getValue(encodedKey); final byte[] foundValue = getValue(encodedKey);
final V result = foundValue == null ? null : valueEncoder.decode(foundValue); final V result = foundValue == null ? null : valueEncoder.decode(foundValue);
valueCache.put(key, result); valueCache.put(key, result);
return result; return result;
} }
private byte[] putValue(final byte[] key, final byte[] value) { private byte[] putValue(final byte[] key, final byte[] value) {
final long rootNodeOffset = readNodeOffsetOfRootNode(); final long rootNodeOffset = readNodeOffsetOfRootNode();
final Stack<PersistentMapDiskNode> parents = new Stack<>(); final Stack<PersistentMapDiskNode> parents = new Stack<>();
return insert(parents, rootNodeOffset, key, value); return insert(parents, rootNodeOffset, key, value);
} }
private byte[] getValue(final byte[] key) { private byte[] getValue(final byte[] key) {
final long rootNodeOffset = readNodeOffsetOfRootNode(); final long rootNodeOffset = readNodeOffsetOfRootNode();
final NodeEntry entry = findNodeEntry(rootNodeOffset, key); final NodeEntry entry = findNodeEntry(rootNodeOffset, key);
return entry == null ? null : entry.getValue(); return entry == null ? null : entry.getValue();
} }
private byte[] insert(final Stack<PersistentMapDiskNode> parents, final long nodeOffest, final byte[] key, private byte[] insert(final Stack<PersistentMapDiskNode> parents, final long nodeOffest, final byte[] key,
final byte[] value) { final byte[] value) {
final PersistentMapDiskNode node = getNode(nodeOffest); final PersistentMapDiskNode node = getNode(nodeOffest);
final NodeEntry entry = node.getNodeEntryTo(key); final NodeEntry entry = node.getNodeEntryTo(key);
if (entry == null || entry.isDataNode()) { if (entry == null || entry.isDataNode()) {
final byte[] oldValue; final byte[] oldValue;
if (entry == null) { if (entry == null) {
oldValue = null; oldValue = null;
} else { } else {
// found a NodeEntry that is either equal to key, or it is at the insertion // found a NodeEntry that is either equal to key, or it is at the insertion
// point // point
final boolean entryIsForKey = entry.equal(key); final boolean entryIsForKey = entry.equal(key);
oldValue = entryIsForKey ? entry.getValue() : null; oldValue = entryIsForKey ? entry.getValue() : null;
// Early exit, if the oldValue equals the new value. // Early exit, if the oldValue equals the new value.
// We do not have to replace the value, because it would not change anything // We do not have to replace the value, because it would not change anything
// (just cause unnecessary write operations). But we return the oldValue so that // (just cause unnecessary write operations). But we return the oldValue so that
// the caller thinks we replaced the value. // the caller thinks we replaced the value.
if (Objects.equals(oldValue, value)) { if (Objects.equals(oldValue, value)) {
return oldValue; return oldValue;
} }
if (entryIsForKey) { if (entryIsForKey) {
node.removeKey(key); node.removeKey(key);
} }
} }
if (node.canAdd(key, value, maxEntriesInNode)) { if (node.canAdd(key, value, maxEntriesInNode)) {
// insert in existing node // insert in existing node
node.addKeyValue(key, value); node.addKeyValue(key, value);
writeNode(node); writeNode(node);
return oldValue; return oldValue;
} else { } else {
// add new node // add new node
// 1. split current node into A and B // 1. split current node into A and B
splitNode(parents, node); splitNode(parents, node);
// 2. insert the value // 2. insert the value
// start from the root, because we might have added a new root node // start from the root, because we might have added a new root node
return putValue(key, value); return putValue(key, value);
} }
} else { } else {
final long childNodeOffset = toNodeOffset(entry); final long childNodeOffset = toNodeOffset(entry);
parents.add(node); parents.add(node);
return insert(parents, childNodeOffset, key, value); return insert(parents, childNodeOffset, key, value);
} }
} }
private PersistentMapDiskNode splitNode(final Stack<PersistentMapDiskNode> parents, private PersistentMapDiskNode splitNode(final Stack<PersistentMapDiskNode> parents,
final PersistentMapDiskNode node) { final PersistentMapDiskNode node) {
// System.out.println("\n\npre split node: " + node + "\n"); // System.out.println("\n\npre split node: " + node + "\n");
final long newBlockOffset = diskStore.allocateBlock(BLOCK_SIZE); final long newBlockOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode newNode = node.split(newBlockOffset); final PersistentMapDiskNode newNode = node.split(newBlockOffset);
final PersistentMapDiskNode parent = parents.isEmpty() ? null : parents.pop(); final PersistentMapDiskNode parent = parents.isEmpty() ? null : parents.pop();
if (parent != null) { if (parent != null) {
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey(); final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
if (parent.canAdd(newNodeKey, newBlockOffset, maxEntriesInNode)) { if (parent.canAdd(newNodeKey, newBlockOffset, maxEntriesInNode)) {
parent.addKeyNodePointer(newNodeKey, newBlockOffset); parent.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parent); writeNode(parent);
writeNode(newNode); writeNode(newNode);
writeNode(node); writeNode(node);
return parent; return parent;
} else { } else {
final PersistentMapDiskNode grandParentNode = splitNode(parents, parent); final PersistentMapDiskNode grandParentNode = splitNode(parents, parent);
final NodeEntry pointerToParentAfterSplit = grandParentNode.getNodeEntryTo(newNodeKey); final NodeEntry pointerToParentAfterSplit = grandParentNode.getNodeEntryTo(newNodeKey);
Preconditions.checkEqual(pointerToParentAfterSplit.isInnerNode(), true, "{0} is pointer to inner node", Preconditions.checkEqual(pointerToParentAfterSplit.isInnerNode(), true, "{0} is pointer to inner node",
pointerToParentAfterSplit); pointerToParentAfterSplit);
final long parentNodeOffset = toNodeOffset(pointerToParentAfterSplit); // the parent we have to add the final long parentNodeOffset = toNodeOffset(pointerToParentAfterSplit); // the parent we have to add the
// newNode to // newNode to
final PersistentMapDiskNode parentNode = getNode(parentNodeOffset); final PersistentMapDiskNode parentNode = getNode(parentNodeOffset);
parentNode.addKeyNodePointer(newNodeKey, newBlockOffset); parentNode.addKeyNodePointer(newNodeKey, newBlockOffset);
writeNode(parentNode); writeNode(parentNode);
writeNode(newNode); writeNode(newNode);
writeNode(node); writeNode(node);
return parentNode; return parentNode;
} }
} else { } else {
// has no parent -> create a new parent (the new parent will also be the new // has no parent -> create a new parent (the new parent will also be the new
// root) // root)
final long newRootNodeOffset = diskStore.allocateBlock(BLOCK_SIZE); final long newRootNodeOffset = diskStore.allocateBlock(BLOCK_SIZE);
final PersistentMapDiskNode rootNode = PersistentMapDiskNode.emptyRootNode(newRootNodeOffset); final PersistentMapDiskNode rootNode = PersistentMapDiskNode.emptyRootNode(newRootNodeOffset);
final byte[] newNodeKey = newNode.getTopNodeEntry().getKey(); final byte[] newNodeKey = newNode.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(newNodeKey, newBlockOffset); rootNode.addKeyNodePointer(newNodeKey, newBlockOffset);
final byte[] oldNodeKey = node.getTopNodeEntry().getKey(); final byte[] oldNodeKey = node.getTopNodeEntry().getKey();
rootNode.addKeyNodePointer(oldNodeKey, node.getNodeOffset()); rootNode.addKeyNodePointer(oldNodeKey, node.getNodeOffset());
writeNode(rootNode); writeNode(rootNode);
writeNode(newNode); writeNode(newNode);
writeNode(node); writeNode(node);
writeNodeOffsetOfRootNode(newRootNodeOffset); writeNodeOffsetOfRootNode(newRootNodeOffset);
return rootNode; return rootNode;
} }
} }
private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) { private NodeEntry findNodeEntry(final long nodeOffest, final byte[] key) {
final PersistentMapDiskNode node = getNode(nodeOffest); final PersistentMapDiskNode node = getNode(nodeOffest);
final var entry = node.getNodeEntryTo(key); final var entry = node.getNodeEntryTo(key);
if (entry == null) { if (entry == null) {
return null; return null;
} else if (entry.isDataNode()) { } else if (entry.isDataNode()) {
if (entry.equal(key)) { if (entry.equal(key)) {
return entry; return entry;
} else { } else {
return null; return null;
} }
} else { } else {
final long childNodeOffset = toNodeOffset(entry); final long childNodeOffset = toNodeOffset(entry);
return findNodeEntry(childNodeOffset, key); return findNodeEntry(childNodeOffset, key);
} }
} }
private long toNodeOffset(final NodeEntry entry) { private long toNodeOffset(final NodeEntry entry) {
Preconditions.checkEqual(entry.isInnerNode(), true); Preconditions.checkEqual(entry.isInnerNode(), true);
return VariableByteEncoder.decodeFirstValue(entry.getValue()); return VariableByteEncoder.decodeFirstValue(entry.getValue());
} }
private PersistentMapDiskNode getNode(final long nodeOffset) { private PersistentMapDiskNode getNode(final long nodeOffset) {
PersistentMapDiskNode node = nodeCache.get(nodeOffset); PersistentMapDiskNode node = nodeCache.get(nodeOffset);
if (node == null) { if (node == null) {
final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffset, BLOCK_SIZE); final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffset, BLOCK_SIZE);
node = PersistentMapDiskNode.parse(nodeOffset, diskBlock); node = PersistentMapDiskNode.parse(nodeOffset, diskBlock);
nodeCache.put(nodeOffset, node); nodeCache.put(nodeOffset, node);
} }
return node; return node;
} }
private void writeNode(final PersistentMapDiskNode node) { private void writeNode(final PersistentMapDiskNode node) {
if (LOGGER.isTraceEnabled()) { if (LOGGER.isTraceEnabled()) {
LOGGER.trace("writing node {}", node.toString(keyEncoder.asDecoder(), valueEncoder.asDecoder())); LOGGER.trace("writing node {}", node.toString(keyEncoder.asDecoder(), valueEncoder.asDecoder()));
} }
final long nodeOffest = node.getNodeOffset(); final long nodeOffest = node.getNodeOffset();
// final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE); // final DiskBlock diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
DiskBlock diskBlock = node.getDiskBlock(); DiskBlock diskBlock = node.getDiskBlock();
if (diskBlock == null) { if (diskBlock == null) {
diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE); diskBlock = diskStore.getDiskBlock(nodeOffest, BLOCK_SIZE);
} }
final byte[] buffer = diskBlock.getBuffer(); final byte[] buffer = diskBlock.getBuffer();
final byte[] newBuffer = node.serialize(); final byte[] newBuffer = node.serialize();
System.arraycopy(newBuffer, 0, buffer, 0, buffer.length); System.arraycopy(newBuffer, 0, buffer, 0, buffer.length);
diskBlock.writeAsync(); diskBlock.writeAsync();
// diskBlock.force(); // makes writing nodes slower by factor 800 (sic!) // diskBlock.force(); // makes writing nodes slower by factor 800 (sic!)
} }
public synchronized void print() { public synchronized void print() {
visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> { visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
final PrintStream writer = System.out; final PrintStream writer = System.out;
final String children = "#" + node.getEntries().size(); final String children = "#" + node.getEntries().size();
writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry writer.println(" ".repeat(depth) + "@" + node.getNodeOffset() + " " + children + " " + nodeEntry
.toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b)))); .toString(b -> String.valueOf(keyEncoder.decode(b)), b -> String.valueOf(valueEncoder.decode(b))));
}); });
} }
public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) { public synchronized void visitNodeEntriesPreOrder(final VisitorCallback visitor) {
final long rootNodeOffset = readNodeOffsetOfRootNode(); final long rootNodeOffset = readNodeOffsetOfRootNode();
visitNodeEntriesPreOrderRecursively(rootNodeOffset, null, visitor, 0); visitNodeEntriesPreOrderRecursively(rootNodeOffset, null, visitor, 0);
} }
private void visitNodeEntriesPreOrderRecursively(final long nodeOffset, final PersistentMapDiskNode parentNode, private void visitNodeEntriesPreOrderRecursively(final long nodeOffset, final PersistentMapDiskNode parentNode,
final VisitorCallback visitor, final int depth) { final VisitorCallback visitor, final int depth) {
final PersistentMapDiskNode node = getNode(nodeOffset); final PersistentMapDiskNode node = getNode(nodeOffset);
for (final NodeEntry child : node.getEntries()) { for (final NodeEntry child : node.getEntries()) {
visitor.visit(node, parentNode, child, depth); visitor.visit(node, parentNode, child, depth);
if (child.isInnerNode()) { if (child.isInnerNode()) {
final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue()); final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue());
visitNodeEntriesPreOrderRecursively(childNodeOffset, node, visitor, depth + 1); visitNodeEntriesPreOrderRecursively(childNodeOffset, node, visitor, depth + 1);
} }
} }
} }
enum VisitByPrefixMode { enum VisitByPrefixMode {
FIND, ITERATE FIND, ITERATE
} }
public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) { public synchronized void visitValues(final K keyPrefix, final Visitor<K, V> visitor) {
final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix); final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix);
final long rootNodeOffset = readNodeOffsetOfRootNode(); final long rootNodeOffset = readNodeOffsetOfRootNode();
iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor); iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor);
} }
private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) { private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) {
final PersistentMapDiskNode node = getNode(nodeOffest); final PersistentMapDiskNode node = getNode(nodeOffest);
// list of children that might contain a key with the keyPrefix // list of children that might contain a key with the keyPrefix
final List<NodeEntry> nodesForPrefix = node.getNodesByPrefix(keyPrefix); final List<NodeEntry> nodesForPrefix = node.getNodesByPrefix(keyPrefix);
for (final NodeEntry entry : nodesForPrefix) { for (final NodeEntry entry : nodesForPrefix) {
if (entry.isDataNode()) { if (entry.isDataNode()) {
final int prefixCompareResult = entry.compareKeyPrefix(keyPrefix); final int prefixCompareResult = entry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) { if (prefixCompareResult == 0) {
if (Arrays.equals(entry.getKey(), MAX_KEY)) { if (Arrays.equals(entry.getKey(), MAX_KEY)) {
continue; continue;
} }
final K key = keyEncoder.decode(entry.getKey()); final K key = keyEncoder.decode(entry.getKey());
final V value = valueEncoder.decode(entry.getValue()); final V value = valueEncoder.decode(entry.getValue());
visitor.visit(key, value); visitor.visit(key, value);
// System.out.println("--> " + key + "=" + value); // System.out.println("--> " + key + "=" + value);
} else if (prefixCompareResult > 0) { } else if (prefixCompareResult > 0) {
break; break;
} }
} else { } else {
final long childNodeOffset = toNodeOffset(entry); final long childNodeOffset = toNodeOffset(entry);
iterateNodeEntryByPrefix(childNodeOffset, keyPrefix, visitor); iterateNodeEntryByPrefix(childNodeOffset, keyPrefix, visitor);
} }
} }
} }
private long readNodeOffsetOfRootNode() { private long readNodeOffsetOfRootNode() {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize()); final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
return diskBlock.getByteBuffer().getLong(0); return diskBlock.getByteBuffer().getLong(0);
} }
private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) { private void writeNodeOffsetOfRootNode(final long newNodeOffsetToRootNode) {
final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize()); final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode); diskBlock.getByteBuffer().putLong(0, newNodeOffsetToRootNode);
diskBlock.force(); diskBlock.force();
} }
} }

386
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java
View File

@@ -42,256 +42,256 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*/ */
public class PersistentMapDiskNode { public class PersistentMapDiskNode {
private final TreeMap<ByteArrayKey, NodeEntry> entries; private final TreeMap<ByteArrayKey, NodeEntry> entries;
private final long nodeOffset; private final long nodeOffset;
private final DiskBlock diskBlock; private final DiskBlock diskBlock;
public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries, public PersistentMapDiskNode(final long nodeOffset, final Collection<NodeEntry> entries,
final DiskBlock diskBlock) { final DiskBlock diskBlock) {
this.nodeOffset = nodeOffset; this.nodeOffset = nodeOffset;
this.diskBlock = diskBlock; this.diskBlock = diskBlock;
this.entries = toMap(entries); this.entries = toMap(entries);
} }
private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) { private static TreeMap<ByteArrayKey, NodeEntry> toMap(final Collection<NodeEntry> nodeEntries) {
final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>(); final TreeMap<ByteArrayKey, NodeEntry> result = new TreeMap<>();
for (final NodeEntry nodeEntry : nodeEntries) { for (final NodeEntry nodeEntry : nodeEntries) {
result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry); result.put(new ByteArrayKey(nodeEntry.getKey()), nodeEntry);
} }
return result; return result;
} }
public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) { public static PersistentMapDiskNode emptyRootNode(final long nodeOffset) {
return new PersistentMapDiskNode(nodeOffset, Collections.emptyList(), null); return new PersistentMapDiskNode(nodeOffset, Collections.emptyList(), null);
} }
public static PersistentMapDiskNode parse(final long nodeOffset, final DiskBlock diskBlock) { public static PersistentMapDiskNode parse(final long nodeOffset, final DiskBlock diskBlock) {
final byte[] data = diskBlock.getBuffer(); final byte[] data = diskBlock.getBuffer();
if (data.length != PersistentMap.BLOCK_SIZE) { if (data.length != PersistentMap.BLOCK_SIZE) {
throw new IllegalStateException( throw new IllegalStateException(
"block size must be " + PersistentMap.BLOCK_SIZE + " but was " + data.length); "block size must be " + PersistentMap.BLOCK_SIZE + " but was " + data.length);
} }
final LongList longs = VariableByteEncoder.decode(data); final LongList longs = VariableByteEncoder.decode(data);
final List<NodeEntry> entries = deserialize(longs, data); final List<NodeEntry> entries = deserialize(longs, data);
return new PersistentMapDiskNode(nodeOffset, entries, diskBlock); return new PersistentMapDiskNode(nodeOffset, entries, diskBlock);
} }
public static List<NodeEntry> deserialize(final LongList keyLengths, final byte[] buffer) { public static List<NodeEntry> deserialize(final LongList keyLengths, final byte[] buffer) {
final List<NodeEntry> entries = new ArrayList<>(); final List<NodeEntry> entries = new ArrayList<>();
if (keyLengths.isEmpty() || keyLengths.get(0) == 0) { if (keyLengths.isEmpty() || keyLengths.get(0) == 0) {
// node is empty -> should only happen for the root node // node is empty -> should only happen for the root node
} else { } else {
final int numEntries = (int) keyLengths.get(0); final int numEntries = (int) keyLengths.get(0);
int offset = PersistentMap.BLOCK_SIZE; int offset = PersistentMap.BLOCK_SIZE;
for (int i = 0; i < numEntries; i++) { for (int i = 0; i < numEntries; i++) {
final int keyLength = (int) keyLengths.get(i * 2 + 1); final int keyLength = (int) keyLengths.get(i * 2 + 1);
final int valueLength = (int) keyLengths.get(i * 2 + 2); final int valueLength = (int) keyLengths.get(i * 2 + 2);
final int valueOffset = offset - valueLength; final int valueOffset = offset - valueLength;
final int keyOffset = valueOffset - keyLength; final int keyOffset = valueOffset - keyLength;
final int typeOffset = keyOffset - 1; final int typeOffset = keyOffset - 1;
final byte typeByte = buffer[typeOffset]; final byte typeByte = buffer[typeOffset];
final byte[] key = Arrays.copyOfRange(buffer, keyOffset, keyOffset + keyLength); final byte[] key = Arrays.copyOfRange(buffer, keyOffset, keyOffset + keyLength);
final byte[] value = Arrays.copyOfRange(buffer, valueOffset, valueOffset + valueLength); final byte[] value = Arrays.copyOfRange(buffer, valueOffset, valueOffset + valueLength);
final NodeEntry entry = new NodeEntry(ValueType.fromByte(typeByte), key, value); final NodeEntry entry = new NodeEntry(ValueType.fromByte(typeByte), key, value);
entries.add(entry); entries.add(entry);
offset = typeOffset; offset = typeOffset;
} }
} }
return entries; return entries;
} }
public byte[] serialize() { public byte[] serialize() {
return serialize(entries); return serialize(entries);
} }
public DiskBlock getDiskBlock() { public DiskBlock getDiskBlock() {
return diskBlock; return diskBlock;
} }
public long getNodeOffset() { public long getNodeOffset() {
return nodeOffset; return nodeOffset;
} }
public NodeEntry getNodeEntryTo(final byte[] key) { public NodeEntry getNodeEntryTo(final byte[] key) {
final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key)); final Entry<ByteArrayKey, NodeEntry> ceilingEntry = entries.ceilingEntry(new ByteArrayKey(key));
return ceilingEntry != null ? ceilingEntry.getValue() : null; return ceilingEntry != null ? ceilingEntry.getValue() : null;
} }
public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) { public List<NodeEntry> getNodesByPrefix(final byte[] keyPrefix) {
final List<NodeEntry> result = new ArrayList<>(); final List<NodeEntry> result = new ArrayList<>();
for (final NodeEntry nodeEntry : entries.values()) { for (final NodeEntry nodeEntry : entries.values()) {
final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix); final int prefixCompareResult = nodeEntry.compareKeyPrefix(keyPrefix);
if (prefixCompareResult == 0) { if (prefixCompareResult == 0) {
// add all entries where keyPrefix is a prefix of the key // add all entries where keyPrefix is a prefix of the key
result.add(nodeEntry); result.add(nodeEntry);
} else if (prefixCompareResult > 0) { } else if (prefixCompareResult > 0) {
// Only add the first entry where the keyPrefix is smaller (as defined by // Only add the first entry where the keyPrefix is smaller (as defined by
// compareKeyPrefix) than the key. // compareKeyPrefix) than the key.
// These are entries that might contain key with the keyPrefix. But only the // These are entries that might contain key with the keyPrefix. But only the
// first of those can really have such keys. // first of those can really have such keys.
result.add(nodeEntry); result.add(nodeEntry);
break; break;
} }
} }
return result; return result;
} }
public void addKeyValue(final byte[] key, final byte[] value) { public void addKeyValue(final byte[] key, final byte[] value) {
addNode(ValueType.VALUE_INLINE, key, value); addNode(ValueType.VALUE_INLINE, key, value);
} }
public void addKeyNodePointer(final byte[] key, final long nodePointer) { public void addKeyNodePointer(final byte[] key, final long nodePointer) {
final byte[] value = VariableByteEncoder.encode(nodePointer); final byte[] value = VariableByteEncoder.encode(nodePointer);
addNode(ValueType.NODE_POINTER, key, value); addNode(ValueType.NODE_POINTER, key, value);
} }
public void addNode(final ValueType valueType, final byte[] key, final byte[] value) { public void addNode(final ValueType valueType, final byte[] key, final byte[] value) {
final NodeEntry entry = new NodeEntry(valueType, key, value); final NodeEntry entry = new NodeEntry(valueType, key, value);
entries.put(new ByteArrayKey(key), entry); entries.put(new ByteArrayKey(key), entry);
} }
public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) { public boolean canAdd(final byte[] key, final long nodeOffset, final int maxEntriesInNode) {
return canAdd(key, VariableByteEncoder.encode(nodeOffset), maxEntriesInNode); return canAdd(key, VariableByteEncoder.encode(nodeOffset), maxEntriesInNode);
} }
public boolean canAdd(final byte[] key, final byte[] value, final int maxEntriesInNode) { public boolean canAdd(final byte[] key, final byte[] value, final int maxEntriesInNode) {
if (entries.size() > maxEntriesInNode) { if (entries.size() > maxEntriesInNode) {
return false; return false;
} else { } else {
final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value); final NodeEntry entry = new NodeEntry(ValueType.VALUE_INLINE, key, value);
final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1); final List<NodeEntry> tmp = new ArrayList<>(entries.size() + 1);
tmp.addAll(entries.values()); tmp.addAll(entries.values());
tmp.add(entry); tmp.add(entry);
// the +1 is for the null-byte terminator of the prefix // the +1 is for the null-byte terminator of the prefix
return neededBytesTotal(tmp) + 1 <= PersistentMap.BLOCK_SIZE; return neededBytesTotal(tmp) + 1 <= PersistentMap.BLOCK_SIZE;
} }
} }
public void removeKey(final byte[] key) { public void removeKey(final byte[] key) {
entries.remove(new ByteArrayKey(key)); entries.remove(new ByteArrayKey(key));
} }
public List<NodeEntry> getEntries() { public List<NodeEntry> getEntries() {
return new ArrayList<>(entries.values()); return new ArrayList<>(entries.values());
} }
public void clear() { public void clear() {
entries.clear(); entries.clear();
} }
@Override @Override
public String toString() { public String toString() {
return "@" + nodeOffset + ": " return "@" + nodeOffset + ": "
+ String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList())); + String.join("\n", entries.values().stream().map(NodeEntry::toString).collect(Collectors.toList()));
} }
public <K,V> String toString(Function<byte[], K> keyDecoder, Function<byte[], V> valueDecoder) {
StringBuilder result = new StringBuilder();
result.append("@");
result.append(nodeOffset);
result.append(": ");
for (NodeEntry e : entries.values()) {
String s = e.toString(keyDecoder, valueDecoder);
result.append("\n");
result.append(s);
}
return result.toString();
}
public NodeEntry getTopNodeEntry() { public <K, V> String toString(Function<byte[], K> keyDecoder, Function<byte[], V> valueDecoder) {
return entries.lastEntry().getValue(); StringBuilder result = new StringBuilder();
} result.append("@");
result.append(nodeOffset);
result.append(": ");
for (NodeEntry e : entries.values()) {
String s = e.toString(keyDecoder, valueDecoder);
result.append("\n");
result.append(s);
}
public PersistentMapDiskNode split(final long newBlockOffset) { return result.toString();
}
final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values()); public NodeEntry getTopNodeEntry() {
return entries.lastEntry().getValue();
}
final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2)); public PersistentMapDiskNode split(final long newBlockOffset) {
final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
entries.clear(); final List<NodeEntry> entriesAsCollection = new ArrayList<>(entries.values());
entries.putAll(toMap(rightEntries));
return new PersistentMapDiskNode(newBlockOffset, leftEntries, null); final var leftEntries = new ArrayList<>(entriesAsCollection.subList(0, entriesAsCollection.size() / 2));
} final var rightEntries = new ArrayList<>(
entriesAsCollection.subList(entriesAsCollection.size() / 2, entriesAsCollection.size()));
public static int neededBytesTotal(final List<NodeEntry> entries) { entries.clear();
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE]; entries.putAll(toMap(rightEntries));
final int usedBytes = serializePrefix(entries, buffer); return new PersistentMapDiskNode(newBlockOffset, leftEntries, null);
}
return usedBytes + NodeEntry.neededBytes(entries); public static int neededBytesTotal(final List<NodeEntry> entries) {
} final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) { final int usedBytes = serializePrefix(entries, buffer);
final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entriesAsCollection, buffer);
// the +1 is for the null-byte terminator of the prefix return usedBytes + NodeEntry.neededBytes(entries);
Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE, }
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
serializeIntoFromTail(entriesAsCollection, buffer); private static byte[] serialize(final Map<ByteArrayKey, NodeEntry> entries) {
return buffer; final byte[] buffer = new byte[PersistentMap.BLOCK_SIZE];
} final Collection<NodeEntry> entriesAsCollection = entries.values();
final int usedBytes = serializePrefix(entriesAsCollection, buffer);
private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) { // the +1 is for the null-byte terminator of the prefix
final LongList longs = serializeKeyLengths(entries); Preconditions.checkGreaterOrEqual(PersistentMap.BLOCK_SIZE,
usedBytes + 1 + NodeEntry.neededBytes(entriesAsCollection),
"The node is too big. It cannot be encoded into " + PersistentMap.BLOCK_SIZE + " bytes.");
final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0); serializeIntoFromTail(entriesAsCollection, buffer);
return usedBytes; return buffer;
} }
private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) { private static int serializePrefix(final Collection<NodeEntry> entries, final byte[] buffer) {
final var keyLengths = new LongList(); final LongList longs = serializeKeyLengths(entries);
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
keyLengths.add(nodeEntry.getKey().length);
keyLengths.add(nodeEntry.getValue().length);
}
return keyLengths; final int usedBytes = VariableByteEncoder.encodeInto(longs, buffer, 0);
} return usedBytes;
}
private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) { private static LongList serializeKeyLengths(final Collection<NodeEntry> entries) {
final var keyLengths = new LongList();
keyLengths.add(entries.size());
for (final NodeEntry nodeEntry : entries) {
keyLengths.add(nodeEntry.getKey().length);
keyLengths.add(nodeEntry.getValue().length);
}
int offset = buffer.length; return keyLengths;
}
for (final var entry : entries) { private static void serializeIntoFromTail(final Collection<NodeEntry> entries, final byte[] buffer) {
final byte[] valueBytes = entry.getValue();
final byte[] keyBytes = entry.getKey();
final int offsetValue = offset - valueBytes.length; int offset = buffer.length;
final int offsetKey = offsetValue - keyBytes.length;
final int offsetType = offsetKey - 1;
System.arraycopy(valueBytes, 0, buffer, offsetValue, valueBytes.length); for (final var entry : entries) {
System.arraycopy(keyBytes, 0, buffer, offsetKey, keyBytes.length); final byte[] valueBytes = entry.getValue();
buffer[offsetType] = entry.getType().asByte(); final byte[] keyBytes = entry.getKey();
offset = offsetType; final int offsetValue = offset - valueBytes.length;
} final int offsetKey = offsetValue - keyBytes.length;
} final int offsetType = offsetKey - 1;
System.arraycopy(valueBytes, 0, buffer, offsetValue, valueBytes.length);
System.arraycopy(keyBytes, 0, buffer, offsetKey, keyBytes.length);
buffer[offsetType] = entry.getType().asByte();
offset = offsetType;
}
}
} }

2
block-storage/src/main/java/org/lucares/pdb/map/Visitor.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.map; package org.lucares.pdb.map;
public interface Visitor<K, V> { public interface Visitor<K, V> {
void visit(K key, V value); void visit(K key, V value);
} }

168
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -25,110 +25,110 @@ import org.testng.annotations.Test;
@Test @Test
public class BSFileTest { public class BSFileTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void testBlockStorage() throws Exception { public void testBlockStorage() throws Exception {
final Path file = dataDirectory.resolve("data.int.db"); final Path file = dataDirectory.resolve("data.int.db");
final int numLongs = 1000; final int numLongs = 1000;
long blockOffset = -1; long blockOffset = -1;
long start = System.nanoTime(); long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) { try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
blockOffset = bsFile.getRootBlockOffset(); try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
for (long i = 0; i < numLongs / 2; i++) { blockOffset = bsFile.getRootBlockOffset();
bsFile.append(i);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE)) {
for (long i = numLongs / 2; i < numLongs; i++) { for (long i = 0; i < numLongs / 2; i++) {
bsFile.append(i); bsFile.append(i);
} }
} }
} try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE)) {
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime(); for (long i = numLongs / 2; i < numLongs; i++) {
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) { bsFile.append(i);
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE); }
final LongList actualLongs = bsFile.asLongList(); }
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1); }
Assert.assertEquals(actualLongs, expectedLongs); System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
public void testBlockStorageMultithreading() throws Exception { start = System.nanoTime();
final ExecutorService pool = Executors.newCachedThreadPool(); try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
final Path file = dataDirectory.resolve("data.int.db"); public void testBlockStorageMultithreading() throws Exception {
final ExecutorService pool = Executors.newCachedThreadPool();
final int threads = 50; final Path file = dataDirectory.resolve("data.int.db");
final int values = 10000;
final Map<Long, LongList> expected = new HashMap<>();
final List<Future<Void>> futures = new ArrayList<>();
final long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
for (int i = 0; i < threads; i++) { final int threads = 50;
final Future<Void> future = pool.submit(() -> { final int values = 10000;
final ThreadLocalRandom random = ThreadLocalRandom.current(); final Map<Long, LongList> expected = new HashMap<>();
final LongList listOfValues = new LongList(); final List<Future<Void>> futures = new ArrayList<>();
final long start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) { for (int i = 0; i < threads; i++) {
final Future<Void> future = pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current();
final LongList listOfValues = new LongList();
for (int j = 0; j < values; j++) { try (BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
// will produce 1,2 and 3 byte sequences when encoded for (int j = 0; j < values; j++) {
final long value = random.nextLong(32768);
listOfValues.add(value);
bsFile.append(value);
}
expected.put(bsFile.getRootBlockOffset(), listOfValues);
}
return null; // will produce 1,2 and 3 byte sequences when encoded
}); final long value = random.nextLong(32768);
futures.add(future); listOfValues.add(value);
} bsFile.append(value);
}
expected.put(bsFile.getRootBlockOffset(), listOfValues);
}
for (final Future<Void> future : futures) { return null;
future.get(); });
} futures.add(future);
}
pool.shutdown(); for (final Future<Void> future : futures) {
pool.awaitTermination(5, TimeUnit.MINUTES); future.get();
} }
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
// verification pool.shutdown();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) { pool.awaitTermination(5, TimeUnit.MINUTES);
for (final Entry<Long, LongList> entry : expected.entrySet()) { }
final long rootBlockNumber = entry.getKey(); System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
final LongList expectedValues = entry.getValue();
try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds, NullCustomizer.INSTANCE)) { // verification
final LongList actualLongs = bsFile.asLongList(); try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final LongList expectedLongs = expectedValues; for (final Entry<Long, LongList> entry : expected.entrySet()) {
Assert.assertEquals(actualLongs, expectedLongs, "for rootBlockNumber=" + rootBlockNumber); final long rootBlockNumber = entry.getKey();
} final LongList expectedValues = entry.getValue();
}
} try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds, NullCustomizer.INSTANCE)) {
} final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = expectedValues;
Assert.assertEquals(actualLongs, expectedLongs, "for rootBlockNumber=" + rootBlockNumber);
}
}
}
}
} }

100
block-storage/src/test/java/org/lucares/pdb/blockstorage/TimeSeriesFileTest.java
View File

@@ -15,70 +15,70 @@ import org.testng.annotations.BeforeMethod;
public class TimeSeriesFileTest { public class TimeSeriesFileTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void testBlockStorageTimeValue() throws Exception { public void testBlockStorageTimeValue() throws Exception {
final Path file = dataDirectory.resolve("data.int.db"); final Path file = dataDirectory.resolve("data.int.db");
final Random random = ThreadLocalRandom.current(); final Random random = ThreadLocalRandom.current();
final int numTimeValuePairs = 1000; final int numTimeValuePairs = 1000;
long blockNumber = -1; long blockNumber = -1;
final LongList expectedLongs = new LongList(); final LongList expectedLongs = new LongList();
long start = System.nanoTime(); long start = System.nanoTime();
long lastEpochMilli = 0; long lastEpochMilli = 0;
// //
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) { try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) { try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockOffset(); blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) { for (long i = 0; i < numTimeValuePairs / 2; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(1000); final long epochMilli = lastEpochMilli + random.nextInt(1000);
final long value = random.nextInt(10000); final long value = random.nextInt(10000);
lastEpochMilli = epochMilli; lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value); bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli); expectedLongs.add(epochMilli);
expectedLongs.add(value); expectedLongs.add(value);
} }
} }
try (final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds)) { try (final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds)) {
for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) { for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(100); final long epochMilli = lastEpochMilli + random.nextInt(100);
final long value = random.nextInt(10000); final long value = random.nextInt(10000);
lastEpochMilli = epochMilli; lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value); bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli); expectedLongs.add(epochMilli);
expectedLongs.add(value); expectedLongs.add(value);
} }
} }
} }
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime(); start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) { try (final DiskStorage ds = new DiskStorage(file, dataDirectory)) {
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds); final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList(); final LongList actualLongs = bsFile.asTimeValueLongList();
Assert.assertEquals(actualLongs, expectedLongs); Assert.assertEquals(actualLongs, expectedLongs);
} }
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
} }
} }

450
block-storage/src/test/java/org/lucares/pdb/diskstorage/DiskStorageTest.java
View File

@@ -18,289 +18,289 @@ import org.testng.annotations.Test;
@Test @Test
public class DiskStorageTest { public class DiskStorageTest {
private static final int BLOCK_SIZE = 512; private static final int BLOCK_SIZE = 512;
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
/** /**
* File systems work with 4096 byte blocks, but we want to work with 512 bytes * File systems work with 4096 byte blocks, but we want to work with 512 bytes
* per block. Does flushing a 512 byte block flush the full 4096 byte block? * per block. Does flushing a 512 byte block flush the full 4096 byte block?
* *
* @throws Exception * @throws Exception
*/ */
@Test(enabled = false) @Test(enabled = false)
public void testFlushingASectorOrABlock() throws Exception { public void testFlushingASectorOrABlock() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
Files.deleteIfExists(databaseFile); Files.deleteIfExists(databaseFile);
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10; final int numBlocks = 10;
allocateBlocks(ds, numBlocks, BLOCK_SIZE); allocateBlocks(ds, numBlocks, BLOCK_SIZE);
final List<DiskBlock> blocks = new ArrayList<>(); final List<DiskBlock> blocks = new ArrayList<>();
// fill the first 16 512-byte blocks // fill the first 16 512-byte blocks
// that is more than on 4096 byte block // that is more than on 4096 byte block
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE); final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock); assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
diskBlock.writeAsync(); diskBlock.writeAsync();
blocks.add(diskBlock); blocks.add(diskBlock);
} }
// now force (aka flush) a block in the middle of the first 4096 byte block // now force (aka flush) a block in the middle of the first 4096 byte block
blocks.get(3).writeAsync(); blocks.get(3).writeAsync();
blocks.get(3).force(); blocks.get(3).force();
System.exit(0); System.exit(0);
// read all blocks again an check what they contain // read all blocks again an check what they contain
// 1. we do this with the existing file channel // 1. we do this with the existing file channel
// this one should see every change, because we wrote them to the file channel // this one should see every change, because we wrote them to the file channel
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE); final DiskBlock diskBlock = ds.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i); assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
blocks.add(diskBlock); blocks.add(diskBlock);
} }
// 2. we read the file from another file channel // 2. we read the file from another file channel
// this one might not see changes made to the first file channel // this one might not see changes made to the first file channel
// //
// But it does see the changes. Most likely, because both channels // But it does see the changes. Most likely, because both channels
// use the same buffers from the operating system. // use the same buffers from the operating system.
try (DiskStorage ds2 = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds2 = new DiskStorage(databaseFile, dataDirectory)) {
for (int i = 0; i < numBlocks; i++) { for (int i = 0; i < numBlocks; i++) {
final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE); final DiskBlock diskBlock = ds2.getDiskBlock(i, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock, (byte) i); assertAllValuesAreEqual(diskBlock, (byte) i);
fill(diskBlock, (byte) i); fill(diskBlock, (byte) i);
blocks.add(diskBlock); blocks.add(diskBlock);
} }
} }
} }
} }
@Test(enabled = true) @Test(enabled = true)
public void testDiskStorage() throws Exception { public void testDiskStorage() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
final ExecutorService pool = Executors.newCachedThreadPool(); final ExecutorService pool = Executors.newCachedThreadPool();
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int numBlocks = 10; final int numBlocks = 10;
final long[] blockOffsets = allocateBlocks(ds, numBlocks, BLOCK_SIZE); final long[] blockOffsets = allocateBlocks(ds, numBlocks, BLOCK_SIZE);
for (final long blockOffset : blockOffsets) { for (final long blockOffset : blockOffsets) {
final long block = blockOffset; final long block = blockOffset;
pool.submit(() -> { pool.submit(() -> {
final ThreadLocalRandom random = ThreadLocalRandom.current(); final ThreadLocalRandom random = ThreadLocalRandom.current();
try { try {
// now read/write random blocks // now read/write random blocks
for (int j = 0; j < 10; j++) { for (int j = 0; j < 10; j++) {
final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE); final DiskBlock diskBlock = ds.getDiskBlock(block, BLOCK_SIZE);
assertAllValuesAreEqual(diskBlock); assertAllValuesAreEqual(diskBlock);
fill(diskBlock, (byte) random.nextInt(127)); fill(diskBlock, (byte) random.nextInt(127));
if (random.nextBoolean()) { if (random.nextBoolean()) {
diskBlock.writeAsync(); diskBlock.writeAsync();
} else { } else {
diskBlock.writeAsync(); diskBlock.writeAsync();
diskBlock.force(); diskBlock.force();
} }
} }
} catch (final Exception e) { } catch (final Exception e) {
e.printStackTrace(); e.printStackTrace();
throw new RuntimeException(e); throw new RuntimeException(e);
} }
}); });
} }
pool.shutdown(); pool.shutdown();
pool.awaitTermination(1, TimeUnit.MINUTES); pool.awaitTermination(1, TimeUnit.MINUTES);
} }
} }
@Test(enabled = true, expectedExceptions = IllegalArgumentException.class) @Test(enabled = true, expectedExceptions = IllegalArgumentException.class)
public void testAllocationSmallerThanMinimalBlockSize() throws Exception { public void testAllocationSmallerThanMinimalBlockSize() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 31; // minimal block size is 32 final int blockSize = 31; // minimal block size is 32
ds.allocateBlock(blockSize); ds.allocateBlock(blockSize);
} }
} }
@Test(enabled = true) @Test(enabled = true)
public void testAllocateAndFreeSingleBlockInFreeList() throws Exception { public void testAllocateAndFreeSingleBlockInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32; final int blockSize = 32;
final long block_8_39 = ds.allocateBlock(blockSize); final long block_8_39 = ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize); final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize); final long block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(block_8_39, 8); Assert.assertEquals(block_8_39, 8);
Assert.assertEquals(block_40_71, 40); Assert.assertEquals(block_40_71, 40);
Assert.assertEquals(block_72_103, 72); Assert.assertEquals(block_72_103, 72);
ds.free(block_40_71, blockSize); ds.free(block_40_71, blockSize);
// should reuse the block we just freed // should reuse the block we just freed
final long actual_block_40_71 = ds.allocateBlock(blockSize); final long actual_block_40_71 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_40_71, 40); Assert.assertEquals(actual_block_40_71, 40);
} }
} }
@Test(enabled = true) @Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksInFreeList() throws Exception { public void testAllocateAndFreeMultipleBlocksInFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32; final int blockSize = 32;
ds.allocateBlock(blockSize); ds.allocateBlock(blockSize);
final long block_40_71 = ds.allocateBlock(blockSize); final long block_40_71 = ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize); final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize); final long block_104_135 = ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize); ds.allocateBlock(blockSize);
ds.free(block_72_103, blockSize); ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize); ds.free(block_104_135, blockSize);
ds.free(block_40_71, blockSize); // the block with the smaller index is freed last, this increases line ds.free(block_40_71, blockSize); // the block with the smaller index is freed last, this increases line
// coverage, because there is a branch for prepending the root node // coverage, because there is a branch for prepending the root node
// should reuse the first block we just freed // should reuse the first block we just freed
// this removes the root node of the free list // this removes the root node of the free list
final long actual_block_40_71 = ds.allocateBlock(blockSize); final long actual_block_40_71 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_40_71, 40); Assert.assertEquals(actual_block_40_71, 40);
// should reuse the second block we just freed // should reuse the second block we just freed
final long actual_block_72_103 = ds.allocateBlock(blockSize); final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, 72); Assert.assertEquals(actual_block_72_103, 72);
// should reuse the third block we just freed // should reuse the third block we just freed
// this removes the last node of the free list // this removes the last node of the free list
final long actual_block_104_135 = ds.allocateBlock(blockSize); final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, 104); Assert.assertEquals(actual_block_104_135, 104);
final long block_168_199 = ds.allocateBlock(blockSize); final long block_168_199 = ds.allocateBlock(blockSize);
Assert.assertEquals(block_168_199, 168); Assert.assertEquals(block_168_199, 168);
} }
} }
@Test(enabled = true) @Test(enabled = true)
public void testAllocateAndFreeInsertFreeNodeInTheMiddleOfTheFreeList() throws Exception { public void testAllocateAndFreeInsertFreeNodeInTheMiddleOfTheFreeList() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSize = 32; final int blockSize = 32;
ds.allocateBlock(blockSize); ds.allocateBlock(blockSize);
ds.allocateBlock(blockSize); ds.allocateBlock(blockSize);
final long block_72_103 = ds.allocateBlock(blockSize); final long block_72_103 = ds.allocateBlock(blockSize);
final long block_104_135 = ds.allocateBlock(blockSize); final long block_104_135 = ds.allocateBlock(blockSize);
final long block_136_167 = ds.allocateBlock(blockSize); final long block_136_167 = ds.allocateBlock(blockSize);
// free the last block first, to increase code coverage // free the last block first, to increase code coverage
ds.free(block_136_167, blockSize); ds.free(block_136_167, blockSize);
ds.free(block_72_103, blockSize); ds.free(block_72_103, blockSize);
ds.free(block_104_135, blockSize); ds.free(block_104_135, blockSize);
// the first free block is re-used // the first free block is re-used
final long actual_block_72_103 = ds.allocateBlock(blockSize); final long actual_block_72_103 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_72_103, block_72_103); Assert.assertEquals(actual_block_72_103, block_72_103);
final long actual_block_104_135 = ds.allocateBlock(blockSize); final long actual_block_104_135 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_104_135, block_104_135); Assert.assertEquals(actual_block_104_135, block_104_135);
final long actual_block_136_167 = ds.allocateBlock(blockSize); final long actual_block_136_167 = ds.allocateBlock(blockSize);
Assert.assertEquals(actual_block_136_167, block_136_167); Assert.assertEquals(actual_block_136_167, block_136_167);
} }
} }
@Test(enabled = true) @Test(enabled = true)
public void testAllocateAndFreeMultipleBlocksWithDifferentSizes() throws Exception { public void testAllocateAndFreeMultipleBlocksWithDifferentSizes() throws Exception {
final Path databaseFile = dataDirectory.resolve("db.ds"); final Path databaseFile = dataDirectory.resolve("db.ds");
try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) { try (DiskStorage ds = new DiskStorage(databaseFile, dataDirectory)) {
final int blockSizeSmall = 32; final int blockSizeSmall = 32;
final int blockSizeBig = 64; final int blockSizeBig = 64;
ds.allocateBlock(blockSizeSmall); ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall); ds.allocateBlock(blockSizeSmall);
final long big_block_72_103 = ds.allocateBlock(blockSizeBig); final long big_block_72_103 = ds.allocateBlock(blockSizeBig);
final long small_block_136_167 = ds.allocateBlock(blockSizeSmall); final long small_block_136_167 = ds.allocateBlock(blockSizeSmall);
ds.allocateBlock(blockSizeSmall); ds.allocateBlock(blockSizeSmall);
ds.free(big_block_72_103, blockSizeBig); ds.free(big_block_72_103, blockSizeBig);
ds.free(small_block_136_167, blockSizeSmall); ds.free(small_block_136_167, blockSizeSmall);
final long actual_small_block_136_167 = ds.allocateBlock(blockSizeSmall); final long actual_small_block_136_167 = ds.allocateBlock(blockSizeSmall);
Assert.assertEquals(actual_small_block_136_167, small_block_136_167); Assert.assertEquals(actual_small_block_136_167, small_block_136_167);
} }
} }
private void assertAllValuesAreEqual(final DiskBlock diskBlock, final byte expectedVal) { private void assertAllValuesAreEqual(final DiskBlock diskBlock, final byte expectedVal) {
final byte[] buffer = diskBlock.getBuffer(); final byte[] buffer = diskBlock.getBuffer();
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
if (expectedVal != buffer[i]) { if (expectedVal != buffer[i]) {
System.err.println( System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i); "block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expectedVal + " at " + i);
break; break;
} }
} }
} }
private void assertAllValuesAreEqual(final DiskBlock diskBlock) { private void assertAllValuesAreEqual(final DiskBlock diskBlock) {
final byte[] buffer = diskBlock.getBuffer(); final byte[] buffer = diskBlock.getBuffer();
final byte expected = buffer[0]; final byte expected = buffer[0];
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
if (expected != buffer[i]) { if (expected != buffer[i]) {
System.err.println( System.err.println(
"block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i); "block " + diskBlock.getBlockOffset() + " " + buffer[i] + " != " + expected + " at " + i);
break; break;
} }
} }
} }
private void fill(final DiskBlock diskBlock, final byte val) { private void fill(final DiskBlock diskBlock, final byte val) {
final byte[] buffer = diskBlock.getBuffer(); final byte[] buffer = diskBlock.getBuffer();
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
buffer[i] = val; buffer[i] = val;
} }
} }
private long[] allocateBlocks(final DiskStorage ds, final int numNewBlocks, final int blockSize) private long[] allocateBlocks(final DiskStorage ds, final int numNewBlocks, final int blockSize)
throws IOException { throws IOException {
final long[] result = new long[numNewBlocks]; final long[] result = new long[numNewBlocks];
for (int i = 0; i < numNewBlocks; i++) { for (int i = 0; i < numNewBlocks; i++) {
final long blockOffset = ds.allocateBlock(blockSize); final long blockOffset = ds.allocateBlock(blockSize);
result[i] = blockOffset; result[i] = blockOffset;
} }
return result; return result;
} }
} }

132
block-storage/src/test/java/org/lucares/pdb/map/CsvTestDataCreator.java
View File

@@ -15,79 +15,79 @@ import java.util.concurrent.ThreadLocalRandom;
public class CsvTestDataCreator { public class CsvTestDataCreator {
private static final List<String> PODS = Arrays.asList("vapbrewe01", "vapfinra01", "vapondem01", "vapondem02", private static final List<String> PODS = Arrays.asList("vapbrewe01", "vapfinra01", "vapondem01", "vapondem02",
"vapondem03", "vapondem04", "vapnyse01", "vapnorto01", "vapfackb01", "vaprjrey01", "vadtrans01", "vapondem03", "vapondem04", "vapnyse01", "vapnorto01", "vapfackb01", "vaprjrey01", "vadtrans01",
"vadaxcel09", "vadaxcel66"); "vadaxcel09", "vadaxcel66");
private static final List<String> HOSTS = new ArrayList<>(); private static final List<String> HOSTS = new ArrayList<>();
private static final List<String> CLASSES = Arrays.asList("AuditLog", "Brava", "Collection", "Folder", "Field", private static final List<String> CLASSES = Arrays.asList("AuditLog", "Brava", "Collection", "Folder", "Field",
"Tagging", "Arrangment", "Review", "Production", "ProductionExport", "View", "Jobs", "Navigation", "Tagging", "Arrangment", "Review", "Production", "ProductionExport", "View", "Jobs", "Navigation",
"RecentNavigation", "Entity", "Search", "Tasks", "PcWorkflow", "Batch", "Matter"); "RecentNavigation", "Entity", "Search", "Tasks", "PcWorkflow", "Batch", "Matter");
private static final List<String> ENDPOINTS = Arrays.asList("create", "remove", "update", "delete", "createBulk", private static final List<String> ENDPOINTS = Arrays.asList("create", "remove", "update", "delete", "createBulk",
"removeBulk", "deleteBulk", "list", "index", "listing", "all"); "removeBulk", "deleteBulk", "list", "index", "listing", "all");
private static final List<String> METHODS = new ArrayList<>(); private static final List<String> METHODS = new ArrayList<>();
private static final List<String> PROJECTS = new ArrayList<>(); private static final List<String> PROJECTS = new ArrayList<>();
private static final List<String> SOURCE = Arrays.asList("web", "service", "metrics"); private static final List<String> SOURCE = Arrays.asList("web", "service", "metrics");
private static final List<String> BUILDS = new ArrayList<>(); private static final List<String> BUILDS = new ArrayList<>();
static { static {
for (int i = 0; i < 500; i++) { for (int i = 0; i < 500; i++) {
BUILDS.add("AXC_5.15_" + i); BUILDS.add("AXC_5.15_" + i);
} }
for (int i = 0; i < 500; i++) { for (int i = 0; i < 500; i++) {
HOSTS.add(UUID.randomUUID().toString().substring(1, 16)); HOSTS.add(UUID.randomUUID().toString().substring(1, 16));
PROJECTS.add(UUID.randomUUID().toString().substring(1, 16) + "_Review"); PROJECTS.add(UUID.randomUUID().toString().substring(1, 16) + "_Review");
} }
for (final String clazz : CLASSES) { for (final String clazz : CLASSES) {
for (final String endpoint : ENDPOINTS) { for (final String endpoint : ENDPOINTS) {
METHODS.add(clazz + "Service." + endpoint); METHODS.add(clazz + "Service." + endpoint);
METHODS.add(clazz + "Controller." + endpoint); METHODS.add(clazz + "Controller." + endpoint);
} }
} }
} }
public static void main(final String[] args) throws IOException { public static void main(final String[] args) throws IOException {
final Path testdataFile = Files.createTempFile("testData", ".csv"); final Path testdataFile = Files.createTempFile("testData", ".csv");
final ThreadLocalRandom r = ThreadLocalRandom.current(); final ThreadLocalRandom r = ThreadLocalRandom.current();
int lines = 0; int lines = 0;
try (FileWriter writer = new FileWriter(testdataFile.toFile())) { try (FileWriter writer = new FileWriter(testdataFile.toFile())) {
writer.append("@timestamp,duration,pod,host,method,project,source,build\n"); writer.append("@timestamp,duration,pod,host,method,project,source,build\n");
for (lines = 0; lines < 1_000_000; lines++) { for (lines = 0; lines < 1_000_000; lines++) {
final String timestamp = Instant.ofEpochMilli(r.nextLong(1234567890L, 12345678901L)) final String timestamp = Instant.ofEpochMilli(r.nextLong(1234567890L, 12345678901L))
.atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); .atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
final String duration = String.valueOf(r.nextInt(10000)); final String duration = String.valueOf(r.nextInt(10000));
final String pod = PODS.get(r.nextInt(PODS.size())); final String pod = PODS.get(r.nextInt(PODS.size()));
final String host = HOSTS.get(r.nextInt(HOSTS.size())); final String host = HOSTS.get(r.nextInt(HOSTS.size()));
final String method = METHODS.get(r.nextInt(METHODS.size())); final String method = METHODS.get(r.nextInt(METHODS.size()));
final String project = PROJECTS.get(r.nextInt(PROJECTS.size())); final String project = PROJECTS.get(r.nextInt(PROJECTS.size()));
final String source = SOURCE.get(r.nextInt(SOURCE.size())); final String source = SOURCE.get(r.nextInt(SOURCE.size()));
final String build = BUILDS.get(r.nextInt(BUILDS.size())); final String build = BUILDS.get(r.nextInt(BUILDS.size()));
writer.append(timestamp); writer.append(timestamp);
writer.append(","); writer.append(",");
writer.append(duration); writer.append(duration);
writer.append(","); writer.append(",");
writer.append(pod); writer.append(pod);
writer.append(","); writer.append(",");
writer.append(host); writer.append(host);
writer.append(","); writer.append(",");
writer.append(method); writer.append(method);
writer.append(","); writer.append(",");
writer.append(project); writer.append(project);
writer.append(","); writer.append(",");
writer.append(source); writer.append(source);
writer.append(","); writer.append(",");
writer.append(build); writer.append(build);
writer.append("\n"); writer.append("\n");
if (lines % 1000 == 0) { if (lines % 1000 == 0) {
System.out.println("lines: " + lines); System.out.println("lines: " + lines);
} }
} }
} }
} }
} }

36
block-storage/src/test/java/org/lucares/pdb/map/NodeEntryTest.java
View File

@@ -11,27 +11,27 @@ import org.testng.annotations.Test;
@Test @Test
public class NodeEntryTest { public class NodeEntryTest {
@DataProvider @DataProvider
public Object[][] providerPrefixCompare() { public Object[][] providerPrefixCompare() {
final List<Object[]> result = new ArrayList<>(); final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { "ab", "abc", -1 }); result.add(new Object[] { "ab", "abc", -1 });
result.add(new Object[] { "abb", "abc", -1 }); result.add(new Object[] { "abb", "abc", -1 });
result.add(new Object[] { "abc", "abc", 0 }); result.add(new Object[] { "abc", "abc", 0 });
result.add(new Object[] { "abcd", "abc", 0 }); result.add(new Object[] { "abcd", "abc", 0 });
result.add(new Object[] { "abd", "abc", 1 }); result.add(new Object[] { "abd", "abc", 1 });
result.add(new Object[] { "abz", "abc", 23 }); result.add(new Object[] { "abz", "abc", 23 });
return result.toArray(Object[][]::new); return result.toArray(Object[][]::new);
} }
@Test(dataProvider = "providerPrefixCompare") @Test(dataProvider = "providerPrefixCompare")
public void testPrefixCompare(final String key, final String prefix, final int expected) { public void testPrefixCompare(final String key, final String prefix, final int expected) {
final NodeEntry nodeEntry = new NodeEntry(ValueType.NODE_POINTER, key.getBytes(StandardCharsets.UTF_8), final NodeEntry nodeEntry = new NodeEntry(ValueType.NODE_POINTER, key.getBytes(StandardCharsets.UTF_8),
new byte[0]); new byte[0]);
final int actual = nodeEntry.compareKeyPrefix(prefix.getBytes(StandardCharsets.UTF_8)); final int actual = nodeEntry.compareKeyPrefix(prefix.getBytes(StandardCharsets.UTF_8));
Assert.assertEquals(actual, expected, key + " ? " + prefix); Assert.assertEquals(actual, expected, key + " ? " + prefix);
} }
} }

36
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapDiskNodeTest.java
View File

@@ -14,29 +14,29 @@ import org.testng.annotations.Test;
@Test @Test
public class PersistentMapDiskNodeTest { public class PersistentMapDiskNodeTest {
public void serializeDeserialize() throws Exception { public void serializeDeserialize() throws Exception {
final List<NodeEntry> entries = new ArrayList<>(); final List<NodeEntry> entries = new ArrayList<>();
entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1")); entries.add(newNode(ValueType.NODE_POINTER, "key1", "value1"));
entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--")); entries.add(newNode(ValueType.VALUE_INLINE, "key2_", "value2--"));
entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---")); entries.add(newNode(ValueType.NODE_POINTER, "key3__", "value3---"));
entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----")); entries.add(newNode(ValueType.VALUE_INLINE, "key4___", "value4----"));
final long nodeOffset = ThreadLocalRandom.current().nextInt(); final long nodeOffset = ThreadLocalRandom.current().nextInt();
final PersistentMapDiskNode node = new PersistentMapDiskNode(nodeOffset, entries, null); final PersistentMapDiskNode node = new PersistentMapDiskNode(nodeOffset, entries, null);
final byte[] buffer = node.serialize(); final byte[] buffer = node.serialize();
final ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); final ByteBuffer byteBuffer = ByteBuffer.wrap(buffer);
final PersistentMapDiskNode actualNode = PersistentMapDiskNode.parse(nodeOffset, final PersistentMapDiskNode actualNode = PersistentMapDiskNode.parse(nodeOffset,
new DiskBlock(nodeOffset, byteBuffer)); new DiskBlock(nodeOffset, byteBuffer));
Assert.assertEquals(actualNode.getEntries(), entries); Assert.assertEquals(actualNode.getEntries(), entries);
} }
private static NodeEntry newNode(final ValueType type, final String key, final String value) { private static NodeEntry newNode(final ValueType type, final String key, final String value) {
return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8), return new NodeEntry(ValueType.VALUE_INLINE, key.getBytes(StandardCharsets.UTF_8),
value.getBytes(StandardCharsets.UTF_8)); value.getBytes(StandardCharsets.UTF_8));
} }
} }

711
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapTest.java
View File

@@ -24,368 +24,369 @@ import org.testng.annotations.Test;
@Test @Test
public class PersistentMapTest { public class PersistentMapTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void testSingleValue() throws Exception { public void testSingleValue() throws Exception {
final Path file = dataDirectory.resolve("map.db"); final Path file = dataDirectory.resolve("map.db");
final String value = "value1"; final String value = "value1";
final String key = "key1"; final String key = "key1";
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory, PersistentMap.STRING_CODER, try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER)) { PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
Assert.assertNull(map.getValue(key)); Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value)); Assert.assertNull(map.putValue(key, value));
Assert.assertEquals(map.getValue(key), value); Assert.assertEquals(map.getValue(key), value);
} }
try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,PersistentMap.STRING_CODER, try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER)) { PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
Assert.assertEquals(map.getValue(key), value); Assert.assertEquals(map.getValue(key), value);
} }
} }
@Test(invocationCount = 1) @Test(invocationCount = 1)
public void testManyValues() throws Exception { public void testManyValues() throws Exception {
final Path file = dataDirectory.resolve("map.db"); final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>(); final var insertedValues = new HashMap<String, String>();
final Random rnd = new Random(1); final Random rnd = new Random(1);
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER, try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER)) { PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
map.setMaxEntriesInNode(2); map.setMaxEntriesInNode(2);
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
// System.out.println("\n\ninserting: " + i); // System.out.println("\n\ninserting: " + i);
final UUID nextUUID = new UUID(rnd.nextLong(), rnd.nextLong()); final UUID nextUUID = new UUID(rnd.nextLong(), rnd.nextLong());
final String key = nextUUID.toString() + "__" + i; final String key = nextUUID.toString() + "__" + i;
final String value = "long value to waste some bytes " + i + "__" final String value = "long value to waste some bytes " + i + "__"
+ UUID.randomUUID().toString().repeat(1); + UUID.randomUUID().toString().repeat(1);
Assert.assertNull(map.getValue(key)); Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value)); Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value); insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER); // map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final boolean failEarly = false; final boolean failEarly = false;
if (failEarly) { if (failEarly) {
for (final var entry : insertedValues.entrySet()) { for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey()); final String actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print();
} }
Assert.assertEquals(actualValue, entry.getValue(), Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration"); "value for key " + entry.getKey() + " in the " + i + "th iteration");
} }
} }
} }
} }
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER, try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER)) { PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER); // map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger maxDepth = new AtomicInteger(); final AtomicInteger maxDepth = new AtomicInteger();
map.visitNodeEntriesPreOrder( map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> maxDepth.set(Math.max(depth, maxDepth.get()))); (node, parentNode, nodeEntry, depth) -> maxDepth.set(Math.max(depth, maxDepth.get())));
Assert.assertTrue(maxDepth.get() >= 4, Assert.assertTrue(maxDepth.get() >= 4,
"The tree's depth. This test must have at least depth 4, " "The tree's depth. This test must have at least depth 4, "
+ "so that we can be sure that splitting parent nodes works recursively, but was " + "so that we can be sure that splitting parent nodes works recursively, but was "
+ maxDepth.get()); + maxDepth.get());
for (final var entry : insertedValues.entrySet()) { for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey()); final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(), Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations"); "value for key " + entry.getKey() + " after all iterations");
} }
} }
} }
@Test(invocationCount = 1) @Test(invocationCount = 1)
public void testManySmallValues() throws Exception { public void testManySmallValues() throws Exception {
final Path file = dataDirectory.resolve("map.db"); final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>(); final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom(); final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1); rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER, try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) { PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1000; i++) { for (int i = 0; i < 1000; i++) {
// System.out.println("\n\ninserting: " + i); // System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE); final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE); final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getValue(key)); Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value)); Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value); insertedValues.put(key, value);
// map.print(); // map.print();
final boolean failEarly = false; final boolean failEarly = false;
if (failEarly) { if (failEarly) {
for (final var entry : insertedValues.entrySet()) { for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey()); final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) { if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); map.print();
} }
Assert.assertEquals(actualValue, entry.getValue(), Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration"); "value for key " + entry.getKey() + " in the " + i + "th iteration");
} }
} }
} }
} }
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER, try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) { PersistentMap.LONG_CODER)) {
// map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER); // map.print(PersistentMap.LONG_DECODER, PersistentMap.LONG_DECODER);
final AtomicInteger counter = new AtomicInteger(); final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder( map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0)); (node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4, Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, " "number of nodes should be small. Any number larger than 4 indicates, "
+ "that new inner nodes are created even though the existing inner " + "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values"); + "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) { for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey()); final Long actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(), Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " after all iterations"); "value for key " + entry.getKey() + " after all iterations");
} }
} }
} }
@Test(invocationCount = 1)
public void testManyEmptyValues() throws Exception { @Test(invocationCount = 1)
final Path file = dataDirectory.resolve("map.db"); public void testManyEmptyValues() throws Exception {
final var insertedValues = new HashMap<Long, Empty>(); final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Empty>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1); final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) { try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) {
for (int i = 0; i < 1500; i++) {
// System.out.println("\n\ninserting: " + i); for (int i = 0; i < 1500; i++) {
// System.out.println("\n\ninserting: " + i);
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Empty value = Empty.INSTANCE; final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
Assert.assertNull(map.getValue(key)); final Empty value = Empty.INSTANCE;
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
insertedValues.put(key, value);
// map.print();
// map.print();
final boolean failEarly = false;
if (failEarly) { final boolean failEarly = false;
for (final var entry : insertedValues.entrySet()) { if (failEarly) {
final Empty actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); if (!Objects.equals(actualValue, entry.getValue())) {
} map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " in the " + i + "th iteration");
} }
} }
} }
}
try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.EMPTY_ENCODER)) { try (final PersistentMap<Long, Empty> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
map.print(); PersistentMap.EMPTY_ENCODER)) {
final AtomicInteger counter = new AtomicInteger(); map.print();
map.visitNodeEntriesPreOrder( final AtomicInteger counter = new AtomicInteger();
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0)); map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
Assert.assertEquals(counter.get(), 4,
"number of nodes should be small. Any number larger than 4 indicates, " Assert.assertEquals(counter.get(), 4,
+ "that new inner nodes are created even though the existing inner " "number of nodes should be small. Any number larger than 4 indicates, "
+ "nodes could hold the values"); + "that new inner nodes are created even though the existing inner "
+ "nodes could hold the values");
for (final var entry : insertedValues.entrySet()) {
final Empty actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
Assert.assertEquals(actualValue, entry.getValue(), final Empty actualValue = map.getValue(entry.getKey());
"value for key " + entry.getKey() + " after all iterations"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " after all iterations");
}
}
} }
}
@Test(invocationCount = 1)
public void testEasyValues() throws Exception { @Test(invocationCount = 1)
final Path file = dataDirectory.resolve("map.db"); public void testEasyValues() throws Exception {
final var insertedValues = new HashMap<String, String>(); final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<String, String>();
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
final Queue<Integer> numbers = new LinkedList<>(Arrays.asList(1, 15, 11, 4, 16, 3, 13));
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) { try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) { final int numbersSize = numbers.size();
for (int i = 0; i < numbersSize; i++) {
final Integer keyNumber = numbers.poll();
// System.out.println("\n\ninserting: " + keyNumber); final Integer keyNumber = numbers.poll();
// System.out.println("\n\ninserting: " + keyNumber);
final String key = "" + keyNumber;
final String value = "value"; final String key = "" + keyNumber;
Assert.assertNull(map.getValue(key)); final String value = "value";
Assert.assertNull(map.getValue(key));
Assert.assertNull(map.putValue(key, value));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
insertedValues.put(key, value);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey());
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " in the " + i + "th iteration");
} }
} }
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) { try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER); PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
// map.print(PersistentMap.STRING_DECODER, PersistentMap.STRING_DECODER);
final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder( final AtomicInteger counter = new AtomicInteger();
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0)); map.visitNodeEntriesPreOrder(
(node, parentNode, nodeEntry, depth) -> counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0));
for (final var entry : insertedValues.entrySet()) {
final String actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
Assert.assertEquals(actualValue, entry.getValue(), final String actualValue = map.getValue(entry.getKey());
"value for key " + entry.getKey() + " after all iterations"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " after all iterations");
}
}
} }
}
@Test
public void testFindAllByPrefix() throws Exception { @Test
final Path file = dataDirectory.resolve("map.db"); public void testFindAllByPrefix() throws Exception {
final Path file = dataDirectory.resolve("map.db");
final Map<String, String> expectedBar = new HashMap<>();
for (int i = 0; i < 100; i++) { final Map<String, String> expectedBar = new HashMap<>();
// the value is a little bit longer to make sure that the values don't fit into for (int i = 0; i < 100; i++) {
// a single leaf node // the value is a little bit longer to make sure that the values don't fit into
expectedBar.put("bar:" + i, "bar:" + i + "__##################################"); // a single leaf node
} expectedBar.put("bar:" + i, "bar:" + i + "__##################################");
}
final Map<String, String> input = new HashMap<>();
input.putAll(expectedBar); final Map<String, String> input = new HashMap<>();
for (int i = 0; i < 500; i++) { input.putAll(expectedBar);
input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); for (int i = 0; i < 500; i++) {
} input.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) { try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
map.putAllValues(input);
} map.putAllValues(input);
}
try (final PersistentMap<String, String> map = new PersistentMap<>(file,dataDirectory, PersistentMap.STRING_CODER,
PersistentMap.STRING_CODER)) { try (final PersistentMap<String, String> map = new PersistentMap<>(file, dataDirectory,
PersistentMap.STRING_CODER, PersistentMap.STRING_CODER)) {
{
final LinkedHashMap<String, String> actualBar = new LinkedHashMap<>(); {
final Visitor<String, String> visitor = (key, value) -> actualBar.put(key, value); final LinkedHashMap<String, String> actualBar = new LinkedHashMap<>();
map.visitValues("bar:", visitor); final Visitor<String, String> visitor = (key, value) -> actualBar.put(key, value);
map.visitValues("bar:", visitor);
Assert.assertEquals(actualBar, expectedBar);
} Assert.assertEquals(actualBar, expectedBar);
} }
} }
}
@Test(invocationCount = 1)
public void testLotsOfValues() throws Exception { @Test(invocationCount = 1)
final Path file = dataDirectory.resolve("map.db"); public void testLotsOfValues() throws Exception {
final var insertedValues = new HashMap<Long, Long>(); final Path file = dataDirectory.resolve("map.db");
final var insertedValues = new HashMap<Long, Long>();
final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1); final SecureRandom rnd = new SecureRandom();
rnd.setSeed(1);
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) { try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) {
for (int i = 0; i < 1_000; i++) {
for (int i = 0; i < 1_000; i++) {
final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE); final Long key = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
final Long value = (long) (rnd.nextGaussian() * Integer.MAX_VALUE);
if (insertedValues.containsKey(key)) {
continue; if (insertedValues.containsKey(key)) {
} continue;
}
Assert.assertNull(map.putValue(key, value));
Assert.assertNull(map.putValue(key, value));
insertedValues.put(key, value);
insertedValues.put(key, value);
final boolean failEarly = false;
if (failEarly) { final boolean failEarly = false;
for (final var entry : insertedValues.entrySet()) { if (failEarly) {
final Long actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
final Long actualValue = map.getValue(entry.getKey());
if (!Objects.equals(actualValue, entry.getValue())) {
map.print(); if (!Objects.equals(actualValue, entry.getValue())) {
} map.print();
}
Assert.assertEquals(actualValue, entry.getValue(),
"value for key " + entry.getKey() + " in the " + i + "th iteration"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " in the " + i + "th iteration");
} }
} }
} }
}
try (final PersistentMap<Long, Long> map = new PersistentMap<>(file,dataDirectory, PersistentMap.LONG_CODER,
PersistentMap.LONG_CODER)) { try (final PersistentMap<Long, Long> map = new PersistentMap<>(file, dataDirectory, PersistentMap.LONG_CODER,
final AtomicInteger counter = new AtomicInteger(); PersistentMap.LONG_CODER)) {
final AtomicInteger maxDepth = new AtomicInteger(); final AtomicInteger counter = new AtomicInteger();
map.visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> { final AtomicInteger maxDepth = new AtomicInteger();
counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0); map.visitNodeEntriesPreOrder((node, parentNode, nodeEntry, depth) -> {
maxDepth.set(Math.max(maxDepth.get(), depth)); counter.addAndGet(nodeEntry.isInnerNode() ? 1 : 0);
}); maxDepth.set(Math.max(maxDepth.get(), depth));
});
final long start = System.nanoTime();
for (final var entry : insertedValues.entrySet()) { final long start = System.nanoTime();
final Long actualValue = map.getValue(entry.getKey()); for (final var entry : insertedValues.entrySet()) {
Assert.assertEquals(actualValue, entry.getValue(), final Long actualValue = map.getValue(entry.getKey());
"value for key " + entry.getKey() + " after all iterations"); Assert.assertEquals(actualValue, entry.getValue(),
} "value for key " + entry.getKey() + " after all iterations");
System.out.println("nodes=" + counter.get() + ", depth=" + maxDepth.get() + ": " }
+ (System.nanoTime() - start) / 1_000_000.0 + "ms"); System.out.println("nodes=" + counter.get() + ", depth=" + maxDepth.get() + ": "
} + (System.nanoTime() - start) / 1_000_000.0 + "ms");
} }
}
} }

348
byte-utils/src/main/java/org/lucares/utils/byteencoder/VariableByteEncoder.java
View File

@@ -25,218 +25,218 @@ import org.lucares.collections.LongList;
*/ */
public class VariableByteEncoder { public class VariableByteEncoder {
public static final long MIN_VALUE = Long.MIN_VALUE / 2 + 1; public static final long MIN_VALUE = Long.MIN_VALUE / 2 + 1;
public static final long MAX_VALUE = Long.MAX_VALUE / 2; public static final long MAX_VALUE = Long.MAX_VALUE / 2;
private static final int MAX_BYTES_PER_VALUE = 10; private static final int MAX_BYTES_PER_VALUE = 10;
private static final int CONTINUATION_BYTE_FLAG = 1 << 7; // 10000000 private static final int CONTINUATION_BYTE_FLAG = 1 << 7; // 10000000
private static final long DATA_BITS = (1 << 7) - 1; // 01111111 private static final long DATA_BITS = (1 << 7) - 1; // 01111111
private static final ThreadLocal<byte[]> SINGLE_VALUE_BUFFER = ThreadLocal private static final ThreadLocal<byte[]> SINGLE_VALUE_BUFFER = ThreadLocal
.withInitial(() -> new byte[MAX_BYTES_PER_VALUE]); .withInitial(() -> new byte[MAX_BYTES_PER_VALUE]);
/** /**
* Encodes time and value into the given buffer. * Encodes time and value into the given buffer.
* <p> * <p>
* If the encoded values do not fit into the buffer, then 0 is returned. The * If the encoded values do not fit into the buffer, then 0 is returned. The
* caller will have to provide a new buffer with more space. * caller will have to provide a new buffer with more space.
* *
* @param value1 first value, (between -(2^62)+1 and 2^62) * @param value1 first value, (between -(2^62)+1 and 2^62)
* @param value2 second value, (between -(2^62)+1 and 2^62) * @param value2 second value, (between -(2^62)+1 and 2^62)
* @param buffer * @param buffer
* @param offsetInBuffer * @param offsetInBuffer
* @return number of bytes appended to the provided buffer * @return number of bytes appended to the provided buffer
*/ */
public static int encodeInto(final long value1, final long value2, final byte[] buffer, final int offsetInBuffer) { public static int encodeInto(final long value1, final long value2, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer; int offset = offsetInBuffer;
final int bytesAdded1 = encodeInto(value1, buffer, offset); final int bytesAdded1 = encodeInto(value1, buffer, offset);
if (bytesAdded1 > 0) { if (bytesAdded1 > 0) {
offset += bytesAdded1; offset += bytesAdded1;
final int bytesAdded2 = encodeInto(value2, buffer, offset); final int bytesAdded2 = encodeInto(value2, buffer, offset);
if (bytesAdded2 > 0) { if (bytesAdded2 > 0) {
// both value fit into the buffer // both value fit into the buffer
// return the number of added bytes // return the number of added bytes
return bytesAdded1 + bytesAdded2; return bytesAdded1 + bytesAdded2;
} else { } else {
// second value did not fit into the buffer, // second value did not fit into the buffer,
// remove the first value // remove the first value
// and return 0 to indicate that the values did not fit // and return 0 to indicate that the values did not fit
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0); Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0; return 0;
} }
} }
// return 0 if the encoded bytes do not fit // return 0 if the encoded bytes do not fit
// the caller will have to provide a new buffer // the caller will have to provide a new buffer
return 0; return 0;
} }
public static LongList decode(final byte[] buffer) { public static LongList decode(final byte[] buffer) {
final LongList result = new LongList(); final LongList result = new LongList();
decodeInto(buffer, result); decodeInto(buffer, result);
return result; return result;
} }
public static int encodeInto(final long value, final byte[] buffer, final int offsetInBuffer) { public static int encodeInto(final long value, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer; int offset = offsetInBuffer;
assert value >= MIN_VALUE : "min encodable value is -2^62+1 = " + MIN_VALUE; assert value >= MIN_VALUE : "min encodable value is -2^62+1 = " + MIN_VALUE;
assert value <= MAX_VALUE : "max encodable value is 2^62 = " + MAX_VALUE; assert value <= MAX_VALUE : "max encodable value is 2^62 = " + MAX_VALUE;
long normVal = encodeIntoPositiveValue(value); long normVal = encodeIntoPositiveValue(value);
try { try {
final long maxFirstByteValue = 127; final long maxFirstByteValue = 127;
while (normVal > maxFirstByteValue) { while (normVal > maxFirstByteValue) {
buffer[offset] = (byte) ((normVal & DATA_BITS) | CONTINUATION_BYTE_FLAG); buffer[offset] = (byte) ((normVal & DATA_BITS) | CONTINUATION_BYTE_FLAG);
offset++; offset++;
normVal = normVal >> 7; // shift by number of value bits normVal = normVal >> 7; // shift by number of value bits
} }
buffer[offset] = (byte) (normVal); buffer[offset] = (byte) (normVal);
return offset - offsetInBuffer + 1; // return number of encoded bytes return offset - offsetInBuffer + 1; // return number of encoded bytes
} catch (final ArrayIndexOutOfBoundsException e) { } catch (final ArrayIndexOutOfBoundsException e) {
// We need more bytes to store the value than are available. // We need more bytes to store the value than are available.
// Reset the bytes we just wrote. // Reset the bytes we just wrote.
Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0); Arrays.fill(buffer, offsetInBuffer, buffer.length, (byte) 0);
return 0; return 0;
} }
} }
private static void decodeInto(final byte[] buffer, final LongList bufferedLongs) { private static void decodeInto(final byte[] buffer, final LongList bufferedLongs) {
for (int i = 0; i < buffer.length; i++) { for (int i = 0; i < buffer.length; i++) {
if (buffer[i] == 0) { if (buffer[i] == 0) {
// no value is encoded to 0 => there are no further values // no value is encoded to 0 => there are no further values
break; break;
} else { } else {
long val = buffer[i] & DATA_BITS; long val = buffer[i] & DATA_BITS;
int shift = 7; int shift = 7;
while (!isLastByte(buffer[i]) && i + 1 < buffer.length) { while (!isLastByte(buffer[i]) && i + 1 < buffer.length) {
val = val | ((buffer[i + 1] & DATA_BITS) << shift); val = val | ((buffer[i + 1] & DATA_BITS) << shift);
i++; i++;
shift += 7; shift += 7;
} }
bufferedLongs.add(decodeIntoSignedValue(val)); bufferedLongs.add(decodeIntoSignedValue(val));
} }
} }
} }
/** /**
* The input value (positive, negative or null) is encoded into a positive * The input value (positive, negative or null) is encoded into a positive
* value. * value.
* *
* <pre> * <pre>
* *
* input: 0 1 -1 2 -2 3 -3 * input: 0 1 -1 2 -2 3 -3
* encoded: 1 2 3 4 5 6 7 * encoded: 1 2 3 4 5 6 7
* </pre> * </pre>
*/ */
private static long encodeIntoPositiveValue(final long value) { private static long encodeIntoPositiveValue(final long value) {
return value > 0 ? value * 2 : (value * -2) + 1; return value > 0 ? value * 2 : (value * -2) + 1;
} }
/** /**
* inverse of {@link #encodeIntoPositiveValue(long)} * inverse of {@link #encodeIntoPositiveValue(long)}
* *
* @param value * @param value
* @return * @return
*/ */
private static long decodeIntoSignedValue(final long value) { private static long decodeIntoSignedValue(final long value) {
return (value / 2) * (value % 2 == 0 ? 1 : -1); return (value / 2) * (value % 2 == 0 ? 1 : -1);
} }
private static boolean isLastByte(final byte b) { private static boolean isLastByte(final byte b) {
return (b & CONTINUATION_BYTE_FLAG) == 0; return (b & CONTINUATION_BYTE_FLAG) == 0;
} }
public static byte[] encode(final long... longs) { public static byte[] encode(final long... longs) {
int neededBytes = 0; int neededBytes = 0;
for (final long l : longs) { for (final long l : longs) {
neededBytes += VariableByteEncoder.neededBytes(l); neededBytes += VariableByteEncoder.neededBytes(l);
} }
final byte[] result = new byte[neededBytes]; final byte[] result = new byte[neededBytes];
final int bytesWritten = encodeInto(longs, result, 0); final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) { if (bytesWritten <= 0) {
throw new IllegalStateException( throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes."); "Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
} }
return result; return result;
} }
public static long decodeFirstValue(final byte[] buffer) { public static long decodeFirstValue(final byte[] buffer) {
int offset = 0; int offset = 0;
long val = buffer[offset] & DATA_BITS; long val = buffer[offset] & DATA_BITS;
int shift = 7; int shift = 7;
while (!isLastByte(buffer[offset]) && offset + 1 < buffer.length) { while (!isLastByte(buffer[offset]) && offset + 1 < buffer.length) {
val = val | ((buffer[offset + 1] & DATA_BITS) << shift); val = val | ((buffer[offset + 1] & DATA_BITS) << shift);
offset++; offset++;
shift += 7; shift += 7;
} }
return decodeIntoSignedValue(val); return decodeIntoSignedValue(val);
} }
public static int encodeInto(final LongList values, final byte[] buffer, final int offsetInBuffer) { public static int encodeInto(final LongList values, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer; int offset = offsetInBuffer;
for (int i = 0; i < values.size(); i++) { for (int i = 0; i < values.size(); i++) {
final long value = values.get(i); final long value = values.get(i);
final int bytesAdded = encodeInto(value, buffer, offset); final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) { if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0); Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0; return 0;
} }
offset += bytesAdded; offset += bytesAdded;
} }
return offset - offsetInBuffer; return offset - offsetInBuffer;
} }
public static int encodeInto(final long[] values, final byte[] buffer, final int offsetInBuffer) { public static int encodeInto(final long[] values, final byte[] buffer, final int offsetInBuffer) {
int offset = offsetInBuffer; int offset = offsetInBuffer;
for (int i = 0; i < values.length; i++) { for (int i = 0; i < values.length; i++) {
final long value = values[i]; final long value = values[i];
final int bytesAdded = encodeInto(value, buffer, offset); final int bytesAdded = encodeInto(value, buffer, offset);
if (bytesAdded <= 0) { if (bytesAdded <= 0) {
Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0); Arrays.fill(buffer, offsetInBuffer, offset, (byte) 0);
return 0; return 0;
} }
offset += bytesAdded; offset += bytesAdded;
} }
return offset - offsetInBuffer; return offset - offsetInBuffer;
} }
public static byte[] encode(final LongList longs) { public static byte[] encode(final LongList longs) {
final int neededBytes = longs.stream().mapToInt(VariableByteEncoder::neededBytes).sum(); final int neededBytes = longs.stream().mapToInt(VariableByteEncoder::neededBytes).sum();
final byte[] result = new byte[neededBytes]; final byte[] result = new byte[neededBytes];
final int bytesWritten = encodeInto(longs, result, 0); final int bytesWritten = encodeInto(longs, result, 0);
if (bytesWritten <= 0) { if (bytesWritten <= 0) {
throw new IllegalStateException( throw new IllegalStateException(
"Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes."); "Did not reserve enough space to store " + longs + ". We reserved only " + neededBytes + " bytes.");
} }
return result; return result;
} }
public static int neededBytes(final long value) { public static int neededBytes(final long value) {
final byte[] buffer = SINGLE_VALUE_BUFFER.get(); final byte[] buffer = SINGLE_VALUE_BUFFER.get();
final int usedBytes = encodeInto(value, buffer, 0); final int usedBytes = encodeInto(value, buffer, 0);
return usedBytes; return usedBytes;
} }
} }

162
byte-utils/src/test/java/org/lucares/utils/byteencoder/VariableByteEncoderTest.java
View File

@@ -14,97 +14,97 @@ import org.testng.annotations.Test;
@Test @Test
public class VariableByteEncoderTest { public class VariableByteEncoderTest {
@DataProvider @DataProvider
public Object[][] providerEncodeDecode() { public Object[][] providerEncodeDecode() {
return new Object[][] { // return new Object[][] { //
// encoded into 1 byte // encoded into 1 byte
{ 10, -5, 5 }, // { 10, -5, 5 }, //
{ 10, 0, 5 }, // { 10, 0, 5 }, //
{ 10, -63, 63 }, // { 10, -63, 63 }, //
// encoded into 2 bytes // encoded into 2 bytes
{ 10, 130, 131 }, // { 10, 130, 131 }, //
// encoded into 3 bytes // encoded into 3 bytes
{ 10, -8191, 8191 }, // { 10, -8191, 8191 }, //
// encoded into n bytes // encoded into n bytes
{ 1, Long.MAX_VALUE / 2 - 4, Long.MAX_VALUE / 2 }, // { 1, Long.MAX_VALUE / 2 - 4, Long.MAX_VALUE / 2 }, //
{ 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2 }, // { 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2 }, //
{ 11, Long.MIN_VALUE / 2 + 1, Long.MIN_VALUE / 2 + 3 }, // { 11, Long.MIN_VALUE / 2 + 1, Long.MIN_VALUE / 2 + 3 }, //
{ 12, Long.MAX_VALUE / 2 - 3, Long.MAX_VALUE / 2 },// { 12, Long.MAX_VALUE / 2 - 3, Long.MAX_VALUE / 2 },//
}; };
} }
@Test(dataProvider = "providerEncodeDecode") @Test(dataProvider = "providerEncodeDecode")
public void testEncodeDecode(final long numValues, final long minValue, final long maxValue) { public void testEncodeDecode(final long numValues, final long minValue, final long maxValue) {
final LongList originalValues = new LongList(); final LongList originalValues = new LongList();
final byte[] buffer = new byte[1024]; final byte[] buffer = new byte[1024];
final AtomicInteger offsetInBuffer = new AtomicInteger(0); final AtomicInteger offsetInBuffer = new AtomicInteger(0);
ThreadLocalRandom.current().longs(numValues, minValue, maxValue).forEachOrdered(value -> { ThreadLocalRandom.current().longs(numValues, minValue, maxValue).forEachOrdered(value -> {
originalValues.add(value); originalValues.add(value);
final int appendedBytes = VariableByteEncoder.encodeInto(value, buffer, offsetInBuffer.get()); final int appendedBytes = VariableByteEncoder.encodeInto(value, buffer, offsetInBuffer.get());
offsetInBuffer.addAndGet(appendedBytes); offsetInBuffer.addAndGet(appendedBytes);
}); });
final LongList actualValues = VariableByteEncoder.decode(buffer); final LongList actualValues = VariableByteEncoder.decode(buffer);
assertEquals(actualValues.toString(), originalValues.toString()); assertEquals(actualValues.toString(), originalValues.toString());
} }
@DataProvider @DataProvider
public Object[][] providerEncodeDecodeOfTwoValues() { public Object[][] providerEncodeDecodeOfTwoValues() {
return new Object[][] { // return new Object[][] { //
{ 12345, 67890, false, 1 }, // first value needs three bytes, it does not fit { 12345, 67890, false, 1 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 2 }, // first value needs three bytes, it does not fit { 12345, 67890, false, 2 }, // first value needs three bytes, it does not fit
{ 12345, 67890, false, 3 }, // first value needs three bytes, second value does not fit { 12345, 67890, false, 3 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 4 }, // first value needs three bytes, second value does not fit { 12345, 67890, false, 4 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, false, 5 }, // first value needs three bytes, second value does not fit { 12345, 67890, false, 5 }, // first value needs three bytes, second value does not fit
{ 12345, 67890, true, 6 }, // both values need three bytes { 12345, 67890, true, 6 }, // both values need three bytes
{ 12345, 67890, true, 10 }, // { 12345, 67890, true, 10 }, //
}; };
} }
@Test(dataProvider = "providerEncodeDecodeOfTwoValues") @Test(dataProvider = "providerEncodeDecodeOfTwoValues")
public void testEncodeDecodeOfTwoValues(final long value1, final long value2, final boolean fits, public void testEncodeDecodeOfTwoValues(final long value1, final long value2, final boolean fits,
final int bufferSize) { final int bufferSize) {
final LongList originalValues = new LongList(); final LongList originalValues = new LongList();
final byte[] buffer = new byte[bufferSize]; final byte[] buffer = new byte[bufferSize];
final int bytesAdded = VariableByteEncoder.encodeInto(value1, value2, buffer, 0); final int bytesAdded = VariableByteEncoder.encodeInto(value1, value2, buffer, 0);
Assert.assertEquals(bytesAdded > 0, fits); Assert.assertEquals(bytesAdded > 0, fits);
if (fits) { if (fits) {
originalValues.addAll(value1, value2); originalValues.addAll(value1, value2);
} else { } else {
Assert.assertEquals(buffer, new byte[bufferSize], Assert.assertEquals(buffer, new byte[bufferSize],
"checks that buffer is resetted after it discovers the values do not fit"); "checks that buffer is resetted after it discovers the values do not fit");
} }
final LongList decodedValues = VariableByteEncoder.decode(buffer); final LongList decodedValues = VariableByteEncoder.decode(buffer);
Assert.assertEquals(decodedValues, originalValues); Assert.assertEquals(decodedValues, originalValues);
} }
@DataProvider @DataProvider
public Object[][] providerNededBytes() { public Object[][] providerNededBytes() {
return new Object[][] { // return new Object[][] { //
{ 0, 1 }, // { 0, 1 }, //
{ -10, 1 }, // { -10, 1 }, //
{ 10, 1 }, // { 10, 1 }, //
{ -63, 1 }, // { -63, 1 }, //
{ 63, 1 }, // { 63, 1 }, //
{ -64, 2 }, // { -64, 2 }, //
{ 64, 2 }, // { 64, 2 }, //
{ -8191, 2 }, // { -8191, 2 }, //
{ 8191, 2 }, // { 8191, 2 }, //
{ -8192, 3 }, // { -8192, 3 }, //
{ 8192, 3 }, // { 8192, 3 }, //
}; };
} }
@Test(dataProvider = "providerNededBytes") @Test(dataProvider = "providerNededBytes")
public void testNeededBytes(final long value, final int expectedNeededBytes) { public void testNeededBytes(final long value, final int expectedNeededBytes) {
final int neededBytes = VariableByteEncoder.neededBytes(value); final int neededBytes = VariableByteEncoder.neededBytes(value);
final byte[] encoded = VariableByteEncoder.encode(value); final byte[] encoded = VariableByteEncoder.encode(value);
Assert.assertEquals(encoded.length, neededBytes); Assert.assertEquals(encoded.length, neededBytes);
} }
} }

94
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -5,60 +5,60 @@ import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.internal.ParititionId; import org.lucares.pdb.datastore.internal.ParititionId;
public class Doc { public class Doc {
private final Tags tags; private final Tags tags;
/** /**
* the block number used by {@link BSFile} * the block number used by {@link BSFile}
*/ */
private final long rootBlockNumber; private final long rootBlockNumber;
private ParititionId partitionId; private ParititionId partitionId;
/** /**
* Initializes a new document. * Initializes a new document.
* <p> * <p>
* The path can be {@code null}. If path is {@code null}, then * The path can be {@code null}. If path is {@code null}, then
* {@code offsetInListingFile} must be set. The path will be initialized lazily * {@code offsetInListingFile} must be set. The path will be initialized lazily
* when needed. * when needed.
* <p> * <p>
* This is used to reduce the memory footprint. * This is used to reduce the memory footprint.
* *
* @param tags * @param tags
* @param offsetInListingFile must be set if {@code path} is {@code null} * @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path. * @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is * @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath} * relative to {@code storageBasePath}
*/ */
public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) { public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) {
this.partitionId = partitionId; this.partitionId = partitionId;
this.tags = tags; this.tags = tags;
this.rootBlockNumber = rootBlockNumber; this.rootBlockNumber = rootBlockNumber;
} }
public ParititionId getPartitionId() { public ParititionId getPartitionId() {
return partitionId; return partitionId;
} }
public Tags getTags() { public Tags getTags() {
return tags; return tags;
} }
/** /**
* the block number used by {@link BSFile} * the block number used by {@link BSFile}
* *
* @return the root block number of this document * @return the root block number of this document
*/ */
public long getRootBlockNumber() { public long getRootBlockNumber() {
return rootBlockNumber; return rootBlockNumber;
} }
public void setPartitionId(final ParititionId partitionId) { public void setPartitionId(final ParititionId partitionId) {
this.partitionId = partitionId; this.partitionId = partitionId;
} }
@Override @Override
public String toString() { public String toString() {
return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]"; return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
} }
} }

8
data-store/src/main/java/org/lucares/pdb/datastore/InvalidValueException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class InvalidValueException extends IllegalArgumentException { public class InvalidValueException extends IllegalArgumentException {
private static final long serialVersionUID = -8707541995666127297L; private static final long serialVersionUID = -8707541995666127297L;
public InvalidValueException(final String msg) { public InvalidValueException(final String msg) {
super(msg); super(msg);
} }
} }

132
data-store/src/main/java/org/lucares/pdb/datastore/PdbFile.java
View File

@@ -14,86 +14,86 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class PdbFile { public class PdbFile {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> { private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private final PartitionDiskStore partitionDiskStorage; private final PartitionDiskStore partitionDiskStorage;
public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) { public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) {
this.partitionDiskStorage = partitionDiskStorage; this.partitionDiskStorage = partitionDiskStorage;
} }
@Override @Override
public Stream<LongList> apply(final PdbFile pdbFile) { public Stream<LongList> apply(final PdbFile pdbFile) {
final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId()); final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId());
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage); final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
return bsFile.streamOfLongLists(); return bsFile.streamOfLongLists();
} }
} }
private final Tags tags; private final Tags tags;
/** /**
* The rootBlockNumber to be used by {@link BSFile} * The rootBlockNumber to be used by {@link BSFile}
*/ */
private final long rootBlockNumber; private final long rootBlockNumber;
private final ParititionId partitionId; private final ParititionId partitionId;
public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) { public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) {
this.partitionId = partitionId; this.partitionId = partitionId;
this.rootBlockNumber = rootBlockNumber; this.rootBlockNumber = rootBlockNumber;
this.tags = tags; this.tags = tags;
} }
public Tags getTags() { public Tags getTags() {
return tags; return tags;
} }
public long getRootBlockNumber() { public long getRootBlockNumber() {
return rootBlockNumber; return rootBlockNumber;
} }
public ParititionId getPartitionId() { public ParititionId getPartitionId() {
return partitionId; return partitionId;
} }
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) { public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) {
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage)); final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
return longStream; return longStream;
} }
@Override @Override
public String toString() { public String toString() {
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId="+partitionId+"]"; return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId=" + partitionId + "]";
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32)); result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode()); result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final PdbFile other = (PdbFile) obj; final PdbFile other = (PdbFile) obj;
if (rootBlockNumber != other.rootBlockNumber) if (rootBlockNumber != other.rootBlockNumber)
return false; return false;
if (tags == null) { if (tags == null) {
if (other.tags != null) if (other.tags != null)
return false; return false;
} else if (!tags.equals(other.tags)) } else if (!tags.equals(other.tags))
return false; return false;
return true; return true;
} }
} }

172
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -1,105 +1,105 @@
package org.lucares.pdb.datastore; package org.lucares.pdb.datastore;
public class Proposal implements Comparable<Proposal> { public class Proposal implements Comparable<Proposal> {
private final String proposedTag; private final String proposedTag;
private final String proposedQuery; private final String proposedQuery;
private final boolean hasResults; private final boolean hasResults;
private final String newQuery; private final String newQuery;
private final int newCaretPosition; private final int newCaretPosition;
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults, public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults,
final String newQuery, final int newCaretPosition) { final String newQuery, final int newCaretPosition) {
super(); super();
this.proposedTag = proposedTag; this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery; this.proposedQuery = proposedQuery;
this.hasResults = hasResults; this.hasResults = hasResults;
this.newQuery = newQuery; this.newQuery = newQuery;
this.newCaretPosition = newCaretPosition; this.newCaretPosition = newCaretPosition;
} }
public Proposal(final Proposal proposal, final boolean hasResults) { public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag; this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery; this.proposedQuery = proposal.proposedQuery;
this.hasResults = hasResults; this.hasResults = hasResults;
this.newQuery = proposal.newQuery; this.newQuery = proposal.newQuery;
this.newCaretPosition = proposal.newCaretPosition; this.newCaretPosition = proposal.newCaretPosition;
} }
public String getProposedTag() { public String getProposedTag() {
return proposedTag; return proposedTag;
} }
public String getProposedQuery() { public String getProposedQuery() {
return proposedQuery; return proposedQuery;
} }
public boolean hasResults() { public boolean hasResults() {
return hasResults; return hasResults;
} }
public String getNewQuery() { public String getNewQuery() {
return newQuery; return newQuery;
} }
public int getNewCaretPosition() { public int getNewCaretPosition() {
return newCaretPosition; return newCaretPosition;
} }
@Override @Override
public String toString() { public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults=" return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults="
+ hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]"; + hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]";
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + (hasResults ? 1231 : 1237); result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + newCaretPosition; result = prime * result + newCaretPosition;
result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode()); result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode());
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode()); result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode()); result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final Proposal other = (Proposal) obj; final Proposal other = (Proposal) obj;
if (hasResults != other.hasResults) if (hasResults != other.hasResults)
return false; return false;
if (newCaretPosition != other.newCaretPosition) if (newCaretPosition != other.newCaretPosition)
return false; return false;
if (newQuery == null) { if (newQuery == null) {
if (other.newQuery != null) if (other.newQuery != null)
return false; return false;
} else if (!newQuery.equals(other.newQuery)) } else if (!newQuery.equals(other.newQuery))
return false; return false;
if (proposedQuery == null) { if (proposedQuery == null) {
if (other.proposedQuery != null) if (other.proposedQuery != null)
return false; return false;
} else if (!proposedQuery.equals(other.proposedQuery)) } else if (!proposedQuery.equals(other.proposedQuery))
return false; return false;
if (proposedTag == null) { if (proposedTag == null) {
if (other.proposedTag != null) if (other.proposedTag != null)
return false; return false;
} else if (!proposedTag.equals(other.proposedTag)) } else if (!proposedTag.equals(other.proposedTag))
return false; return false;
return true; return true;
} }
@Override @Override
public int compareTo(final Proposal o) { public int compareTo(final Proposal o) {
return proposedTag.compareTo(o.getProposedTag()); return proposedTag.compareTo(o.getProposedTag());
} }
} }

8
data-store/src/main/java/org/lucares/pdb/datastore/ReadException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class ReadException extends RuntimeException { public class ReadException extends RuntimeException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public ReadException(final RuntimeException e) { public ReadException(final RuntimeException e) {
super(e); super(e);
} }
} }

20
data-store/src/main/java/org/lucares/pdb/datastore/ReadRuntimeException.java
View File

@@ -2,17 +2,17 @@ package org.lucares.pdb.datastore;
public class ReadRuntimeException extends RuntimeException { public class ReadRuntimeException extends RuntimeException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public ReadRuntimeException(final String message, final Throwable cause) { public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause); super(message, cause);
} }
public ReadRuntimeException(final String message) { public ReadRuntimeException(final String message) {
super(message); super(message);
} }
public ReadRuntimeException(final Throwable cause) { public ReadRuntimeException(final Throwable cause) {
super(cause); super(cause);
} }
} }

14
data-store/src/main/java/org/lucares/pdb/datastore/WriteException.java
View File

@@ -2,14 +2,14 @@ package org.lucares.pdb.datastore;
public class WriteException extends RuntimeException { public class WriteException extends RuntimeException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public WriteException(final String message, final Throwable cause) { public WriteException(final String message, final Throwable cause) {
super(message, cause); super(message, cause);
} }
public WriteException(final Throwable cause) { public WriteException(final Throwable cause) {
super(cause); super(cause);
} }
} }

670
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -39,381 +39,381 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class DataStore implements AutoCloseable { public class DataStore implements AutoCloseable {
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area // area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery"); .getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID"); .getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter"); .getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class); private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
public static final char LISTING_FILE_SEPARATOR = ','; public static final char LISTING_FILE_SEPARATOR = ',';
public static final String SUBDIR_STORAGE = "storage"; public static final String SUBDIR_STORAGE = "storage";
// used to generate doc ids that are // used to generate doc ids that are
// a) unique // a) unique
// b) monotonically increasing (this is, so that we don't have to sort the doc // b) monotonically increasing (this is, so that we don't have to sort the doc
// ids when getting them from the BSFiles) // ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis()); private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
public static Tag TAG_ALL_DOCS = null; public static Tag TAG_ALL_DOCS = null;
private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc; private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc;
private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId; private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId;
private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId; private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId;
private final QueryCompletionIndex queryCompletionIndex; private final QueryCompletionIndex queryCompletionIndex;
// A Doc will never be changed once it is created. Therefore we can cache them // A Doc will never be changed once it is created. Therefore we can cache them
// easily. // easily.
private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000); private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000);
private final HotEntryCache<Tags, PdbWriter> writerCache; private final HotEntryCache<Tags, PdbWriter> writerCache;
private final PartitionDiskStore diskStorage; private final PartitionDiskStore diskStorage;
private final Path storageBasePath; private final Path storageBasePath;
public DataStore(final Path dataDirectory) throws IOException { public DataStore(final Path dataDirectory) throws IOException {
storageBasePath = storageDirectory(dataDirectory); storageBasePath = storageDirectory(dataDirectory);
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath)); Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY); Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put(""); Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it
// must be initialized after the string compressor has been created // must be initialized after the string compressor has been created
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs"); diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs", tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs",
new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER)); new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(), tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(),
PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER)); PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER, docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER,
new DocEncoderDecoder()); new DocEncoderDecoder());
queryCompletionIndex = new QueryCompletionIndex(storageBasePath); queryCompletionIndex = new QueryCompletionIndex(storageBasePath);
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000); writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000);
writerCache.addListener((key, value) -> value.close()); writerCache.addListener((key, value) -> value.close());
} }
private Path keyCompressionFile(final Path dataDirectory) throws IOException { private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv"); return dataDirectory.resolve("keys.csv");
} }
public static Path storageDirectory(final Path dataDirectory) throws IOException { public static Path storageDirectory(final Path dataDirectory) throws IOException {
return dataDirectory.resolve(SUBDIR_STORAGE); return dataDirectory.resolve(SUBDIR_STORAGE);
} }
public void write(final long dateAsEpochMilli, final Tags tags, final long value) { public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli); final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags); final PdbWriter writer = getWriter(partitionId, tags);
writer.write(dateAsEpochMilli, value); writer.write(dateAsEpochMilli, value);
} }
// visible for test // visible for test
QueryCompletionIndex getQueryCompletionIndex() { QueryCompletionIndex getQueryCompletionIndex() {
return queryCompletionIndex; return queryCompletionIndex;
} }
public long createNewFile(final ParititionId partitionId, final Tags tags) { public long createNewFile(final ParititionId partitionId, final Tags tags) {
try { try {
final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE); final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
final long docId = createUniqueDocId(); final long docId = createUniqueDocId();
final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset); final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset);
docIdToDoc.putValue(partitionId, docId, doc); docIdToDoc.putValue(partitionId, docId, doc);
final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId); final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId);
Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags); Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags);
// store mapping from tag to docId, so that we can find all docs for a given tag // store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags()); final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS); ts.add(TAG_ALL_DOCS);
for (final Tag tag : ts) { for (final Tag tag : ts) {
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag); Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
if (diskStoreOffsetForDocIdsOfTag == null) { if (diskStoreOffsetForDocIdsOfTag == null) {
diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE); diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag); tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag);
} }
try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag, try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag,
partitionId)) { partitionId)) {
docIdsOfTag.append(docId); docIdsOfTag.append(docId);
} }
} }
// index the tags, so that we can efficiently find all possible values for a // index the tags, so that we can efficiently find all possible values for a
// field in a query // field in a query
queryCompletionIndex.addTags(partitionId, tags); queryCompletionIndex.addTags(partitionId, tags);
return newFilesRootBlockOffset; return newFilesRootBlockOffset;
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
private long createUniqueDocId() { private long createUniqueDocId() {
return NEXT_DOC_ID.getAndIncrement(); return NEXT_DOC_ID.getAndIncrement();
} }
public List<PdbFile> getFilesForQuery(final Query query) { public List<PdbFile> getFilesForQuery(final Query query) {
final List<Doc> searchResult = search(query); final List<Doc> searchResult = search(query);
if (searchResult.size() > 500_000) { if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results."); throw new IllegalStateException("Too many results.");
} }
final List<PdbFile> result = toPdbFiles(searchResult); final List<PdbFile> result = toPdbFiles(searchResult);
return result; return result;
} }
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) { private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>(searchResult.size()); final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) { for (final Doc document : searchResult) {
final ParititionId partitionId = document.getPartitionId(); final ParititionId partitionId = document.getPartitionId();
final long rootBlockNumber = document.getRootBlockNumber(); final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags(); final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags); final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags);
result.add(pdbFile); result.add(pdbFile);
} }
return result; return result;
} }
public List<Doc> search(final Query query) { public List<Doc> search(final Query query) {
try { try {
final List<Doc> result = new ArrayList<>(); final List<Doc> result = new ArrayList<>();
final PartitionLongList docIdsList = executeQuery(query); final PartitionLongList docIdsList = executeQuery(query);
LOGGER.trace("query {} found {} docs", query, docIdsList.size()); LOGGER.trace("query {} found {} docs", query, docIdsList.size());
final List<Doc> docs = mapDocIdsToDocs(docIdsList); final List<Doc> docs = mapDocIdsToDocs(docIdsList);
result.addAll(docs); result.addAll(docs);
return result; return result;
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
public int count(final Query query) { public int count(final Query query) {
final PartitionLongList docIdsList = executeQuery(query); final PartitionLongList docIdsList = executeQuery(query);
return docIdsList.size(); return docIdsList.size();
} }
public List<String> getAvailableFields(final DateTimeRange dateRange) { public List<String> getAvailableFields(final DateTimeRange dateRange) {
final Set<String> keys = new HashSet<>(); final Set<String> keys = new HashSet<>();
final Tag keyPrefix = new Tag("", ""); // will find everything final Tag keyPrefix = new Tag("", ""); // will find everything
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString())); tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString()));
keys.remove(ALL_DOCS_KEY); keys.remove(ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys); final List<String> result = new ArrayList<>(keys);
Collections.sort(result); Collections.sort(result);
return result; return result;
} }
private PartitionLongList executeQuery(final Query query) { private PartitionLongList executeQuery(final Query query) {
final long start = System.nanoTime(); final long start = System.nanoTime();
synchronized (docIdToDoc) { synchronized (docIdToDoc) {
final Expression expression = QueryLanguageParser.parse(query.getQuery()); final Expression expression = QueryLanguageParser.parse(query.getQuery());
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId, final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId,
diskStorage); diskStorage);
final PartitionLongList docIdsList = expression.visit(visitor); final PartitionLongList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query, EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query,
(System.nanoTime() - start) / 1_000_000.0, docIdsList.size()); (System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList; return docIdsList;
} }
} }
private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException { private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException {
final List<Doc> result = new ArrayList<>(docIdsList.size()); final List<Doc> result = new ArrayList<>(docIdsList.size());
synchronized (docIdToDoc) { synchronized (docIdToDoc) {
final long start = System.nanoTime(); final long start = System.nanoTime();
for (final ParititionId partitionId : docIdsList) { for (final ParititionId partitionId : docIdsList) {
final LongList docIds = docIdsList.get(partitionId); final LongList docIds = docIdsList.get(partitionId);
for (int i = 0; i < docIds.size(); i++) { for (int i = 0; i < docIds.size(); i++) {
final long docId = docIds.get(i); final long docId = docIds.get(i);
final Doc doc = getDocByDocId(partitionId, docId); final Doc doc = getDocByDocId(partitionId, docId);
Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist."); Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist.");
result.add(doc); result.add(doc);
} }
} }
MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(), MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(),
(System.nanoTime() - start) / 1_000_000.0); (System.nanoTime() - start) / 1_000_000.0);
} }
return result; return result;
} }
public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) { public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) {
final Long docId = tagsToDocId.getValue(partitionId, tags); final Long docId = tagsToDocId.getValue(partitionId, tags);
if (docId != null) { if (docId != null) {
final Doc doc = getDocByDocId(partitionId, docId); final Doc doc = getDocByDocId(partitionId, docId);
return Optional.of(doc); return Optional.of(doc);
} }
return Optional.empty(); return Optional.empty();
} }
public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) { public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) {
final List<Doc> result = new ArrayList<>(); final List<Doc> result = new ArrayList<>();
final DatePartitioner datePartitioner = new DatePartitioner(dateRange); final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags); final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags);
for (final Long docId : docIds) { for (final Long docId : docIds) {
if (docId != null) { if (docId != null) {
final Doc doc = getDocByDocId(dateRange, docId); final Doc doc = getDocByDocId(dateRange, docId);
result.add(doc); result.add(doc);
} }
} }
return result; return result;
} }
private Doc getDocByDocId(final ParititionId partitionId, final Long docId) { private Doc getDocByDocId(final ParititionId partitionId, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> { return docIdToDocCache.putIfAbsent(docId, documentId -> {
return docIdToDoc.getValue(partitionId, documentId); return docIdToDoc.getValue(partitionId, documentId);
}); });
} }
private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) { private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> { return docIdToDocCache.putIfAbsent(docId, documentId -> {
final DatePartitioner datePartitioner = new DatePartitioner(dateRange); final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId); final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId);
if (docIds.size() == 1) { if (docIds.size() == 1) {
return docIds.get(0); return docIds.get(0);
} else if (docIds.size() > 1) { } else if (docIds.size() > 1) {
throw new IllegalStateException( throw new IllegalStateException(
"Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds); "Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds);
} }
throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId); throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId);
}); });
} }
public List<Proposal> propose(final QueryWithCaretMarker query) { public List<Proposal> propose(final QueryWithCaretMarker query) {
final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex); final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex);
final List<Proposal> proposals = newProposerParser.propose(query); final List<Proposal> proposals = newProposerParser.propose(query);
LOGGER.debug("Proposals for query {}: {}", query, proposals); LOGGER.debug("Proposals for query {}: {}", query, proposals);
return proposals; return proposals;
} }
public PartitionDiskStore getDiskStorage() { public PartitionDiskStore getDiskStorage() {
return diskStorage; return diskStorage;
} }
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException { private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags)); return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags));
} }
// visible for test // visible for test
long sizeWriterCache() { long sizeWriterCache() {
return writerCache.size(); return writerCache.size();
} }
private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) { private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) {
final Optional<Doc> docsForTags = getByTags(partitionId, tags); final Optional<Doc> docsForTags = getByTags(partitionId, tags);
PdbWriter writer; PdbWriter writer;
if (docsForTags.isPresent()) { if (docsForTags.isPresent()) {
try { try {
final Doc doc = docsForTags.get(); final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags); final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId)); writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
throw new ReadException(e); throw new ReadException(e);
} }
} else { } else {
writer = newPdbWriter(partitionId, tags); writer = newPdbWriter(partitionId, tags);
} }
return writer; return writer;
} }
private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) { private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) {
final long start = System.nanoTime(); final long start = System.nanoTime();
try { try {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags); final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId)); final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}", METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags); (System.nanoTime() - start) / 1_000_000.0, tags);
return result; return result;
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
throw new WriteException(e); throw new WriteException(e);
} }
} }
private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) { private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) {
final long rootBlockNumber = createNewFile(partitionId, tags); final long rootBlockNumber = createNewFile(partitionId, tags);
final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags); final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags);
return result; return result;
} }
@Override @Override
public void close() throws RuntimeIOException { public void close() throws RuntimeIOException {
try { try {
// we cannot simply clear the cache, because the cache implementation (Guava at // we cannot simply clear the cache, because the cache implementation (Guava at
// the time of writing) handles eviction events asynchronously. // the time of writing) handles eviction events asynchronously.
forEachWriter(cachedWriter -> { forEachWriter(cachedWriter -> {
try { try {
cachedWriter.close(); cachedWriter.close();
} catch (final Exception e) { } catch (final Exception e) {
throw new WriteException(e); throw new WriteException(e);
} }
}); });
} finally { } finally {
try { try {
diskStorage.close(); diskStorage.close();
} finally { } finally {
tagToDocsId.close(); tagToDocsId.close();
} }
} }
} }
private void forEachWriter(final Consumer<PdbWriter> consumer) { private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> { writerCache.forEach(writer -> {
try { try {
consumer.accept(writer); consumer.accept(writer);
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e); LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e);
} }
}); });
} }
public void flush() { public void flush() {
forEachWriter(t -> { forEachWriter(t -> {
try { try {
t.flush(); t.flush();
} catch (final Exception e) { } catch (final Exception e) {
throw new WriteException(e); throw new WriteException(e);
} }
}); });
} }
} }

268
data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java
View File

@@ -19,178 +19,178 @@ import org.lucares.pdb.api.DateTimeRange;
public class DateIndexExtension { public class DateIndexExtension {
/** /**
* This date pattern defines the resolution of the date index * This date pattern defines the resolution of the date index
*/ */
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM"); private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
// visible for test // visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>(); static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null); private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null);
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) { static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>(); final Set<String> result = new TreeSet<>();
OffsetDateTime current = dateRange.getStart(); OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) { while (current.isBefore(dateRange.getEnd())) {
result.add(toDateIndexPrefix(current)); result.add(toDateIndexPrefix(current));
current = current.plusMonths(1); current = current.plusMonths(1);
} }
result.add(toDateIndexPrefix(dateRange.getEnd())); result.add(toDateIndexPrefix(dateRange.getEnd()));
return result; return result;
} }
static String toDateIndexPrefix(final OffsetDateTime time) { static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN); return time.format(DATE_PATTERN);
} }
public static ParititionId toPartitionId(final long epochMilli) { public static ParititionId toPartitionId(final long epochMilli) {
String result; String result;
final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get(); final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get();
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
&& lastAccessed.getMaxEpochMilli() >= epochMilli) { && lastAccessed.getMaxEpochMilli() >= epochMilli) {
result = lastAccessed.getDatePrefix(); result = lastAccessed.getDatePrefix();
} else { } else {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli); final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
if (value == null || !value.getValue().contains(epochMilli)) { if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue); DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix(); result = newValue.getDatePrefix();
LAST_ACCESSED.set(newValue); LAST_ACCESSED.set(newValue);
} else { } else {
result = value.getValue().getDatePrefix(); result = value.getValue().getDatePrefix();
LAST_ACCESSED.set(value.getValue()); LAST_ACCESSED.set(value.getValue());
} }
} }
return new ParititionId(result); return new ParititionId(result);
} }
public static String toDateIndexPrefix(final long epochMilli) { public static String toDateIndexPrefix(final long epochMilli) {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli); final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
String result; String result;
if (value == null || !value.getValue().contains(epochMilli)) { if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue); DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix(); result = newValue.getDatePrefix();
} else { } else {
result = value.getValue().getDatePrefix(); result = value.getValue().getDatePrefix();
} }
return result; return result;
} }
/** /**
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final * only for tests, use toPartitionIds(final DateTimeRange dateRange,final
* Collection<? extends PartitionId> availablePartitionIds) instead * Collection<? extends PartitionId> availablePartitionIds) instead
* *
* @param dateRange * @param dateRange
* @return * @return
*/ */
static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) { static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) {
final List<ParititionId> result = new ArrayList<>(); final List<ParititionId> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart(); OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd(); final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0); .withSecond(0).withNano(0);
while (!current.isAfter(end)) { while (!current.isAfter(end)) {
final String id = current.format(DATE_PATTERN); final String id = current.format(DATE_PATTERN);
final ParititionId partitionId = new ParititionId(id); final ParititionId partitionId = new ParititionId(id);
result.add(partitionId); result.add(partitionId);
current = current.plusMonths(1); current = current.plusMonths(1);
} }
return result; return result;
} }
public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange, public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange,
final Collection<? extends ParititionId> availablePartitionIds) { final Collection<? extends ParititionId> availablePartitionIds) {
final Set<ParititionId> result = new LinkedHashSet<>(); final Set<ParititionId> result = new LinkedHashSet<>();
final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli()); final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli());
final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli()); final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli());
for (final ParititionId partitionId : availablePartitionIds) { for (final ParititionId partitionId : availablePartitionIds) {
if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) { if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) {
result.add(partitionId); result.add(partitionId);
} }
} }
return result; return result;
} }
public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) { public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0); final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1); final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
final String datePrefix = date.format(DATE_PATTERN); final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli(); final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli(); final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli); return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
} }
public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) { public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) {
final List<Long> result = new ArrayList<>(); final List<Long> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart(); OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd(); final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0); .withSecond(0).withNano(0);
while (!current.isAfter(end)) { while (!current.isAfter(end)) {
result.add(current.toInstant().toEpochMilli()); result.add(current.toInstant().toEpochMilli());
current = current.plusMonths(1); current = current.plusMonths(1);
} }
return result; return result;
} }
public static ParititionId now() { public static ParititionId now() {
return toPartitionId(System.currentTimeMillis()); return toPartitionId(System.currentTimeMillis());
} }
} }
class DatePrefixAndRange { class DatePrefixAndRange {
private final String datePrefix; private final String datePrefix;
private final long minEpochMilli; private final long minEpochMilli;
private final long maxEpochMilli; private final long maxEpochMilli;
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) { public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super(); super();
this.datePrefix = datePrefix; this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli; this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli; this.maxEpochMilli = maxEpochMilli;
} }
public String getDatePrefix() { public String getDatePrefix() {
return datePrefix; return datePrefix;
} }
public long getMinEpochMilli() { public long getMinEpochMilli() {
return minEpochMilli; return minEpochMilli;
} }
public long getMaxEpochMilli() { public long getMaxEpochMilli() {
return maxEpochMilli; return maxEpochMilli;
} }
public boolean contains(final long epochMilli) { public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli; return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
} }
@Override @Override
public String toString() { public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")"; return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
} }
} }

16
data-store/src/main/java/org/lucares/pdb/datastore/internal/DatePartitioner.java
View File

@@ -6,14 +6,14 @@ import org.lucares.pdb.api.DateTimeRange;
public class DatePartitioner implements PartitionIdSource { public class DatePartitioner implements PartitionIdSource {
private final DateTimeRange dateRange; private final DateTimeRange dateRange;
public DatePartitioner(final DateTimeRange dateRange) { public DatePartitioner(final DateTimeRange dateRange) {
this.dateRange = dateRange; this.dateRange = dateRange;
} }
@Override @Override
public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) { public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) {
return DateIndexExtension.toPartitionIds(dateRange, availablePartitions); return DateIndexExtension.toPartitionIds(dateRange, availablePartitions);
} }
} }

62
data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java
View File

@@ -8,43 +8,43 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> { class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
@Override @Override
public byte[] encode(final Doc doc) { public byte[] encode(final Doc doc) {
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber()); final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().toBytes(); final byte[] tags = doc.getTags().toBytes();
final byte[] result = new byte[rootBlockNumber.length + tags.length]; final byte[] result = new byte[rootBlockNumber.length + tags.length];
System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length); System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length);
System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length); System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length);
return result; return result;
} }
@Override @Override
public Doc decode(final byte[] bytes) { public Doc decode(final byte[] bytes) {
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes); final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber); final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length)); final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(null, tags, rootBlockNumber); return new Doc(null, tags, rootBlockNumber);
} }
@Override @Override
public Doc encodeValue(final Doc v) { public Doc encodeValue(final Doc v) {
return v; return v;
} }
@Override @Override
public Doc decodeValue(final ParititionId partitionId, final Doc t) { public Doc decodeValue(final ParititionId partitionId, final Doc t) {
if (t != null) { if (t != null) {
t.setPartitionId(partitionId); t.setPartitionId(partitionId);
} }
return t; return t;
} }
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] {0}; return new byte[] { 0 };
} }
} }

22
data-store/src/main/java/org/lucares/pdb/datastore/internal/GlobMatcher.java
View File

@@ -7,18 +7,18 @@ import org.lucares.pdb.datastore.lang.GloblikePattern;
public class GlobMatcher { public class GlobMatcher {
private final Pattern pattern; private final Pattern pattern;
public GlobMatcher(final String globlike) { public GlobMatcher(final String globlike) {
pattern = GloblikePattern.globlikeToRegex(globlike); pattern = GloblikePattern.globlikeToRegex(globlike);
} }
public GlobMatcher(final Iterable<String> globlikes) { public GlobMatcher(final Iterable<String> globlikes) {
pattern = GloblikePattern.globlikeToRegex(globlikes); pattern = GloblikePattern.globlikeToRegex(globlikes);
} }
public boolean matches(final String s) { public boolean matches(final String s) {
final Matcher matcher = pattern.matcher(s); final Matcher matcher = pattern.matcher(s);
return matcher.find(); return matcher.find();
} }
} }

106
data-store/src/main/java/org/lucares/pdb/datastore/internal/ParititionId.java
View File

@@ -1,65 +1,65 @@
package org.lucares.pdb.datastore.internal; package org.lucares.pdb.datastore.internal;
public class ParititionId implements Comparable<ParititionId> { public class ParititionId implements Comparable<ParititionId> {
private final String partitionId; private final String partitionId;
/** /**
* Create a new partition id. * Create a new partition id.
* *
* @param partitionId the id, e.g. a time like 201902 (partition for entries of * @param partitionId the id, e.g. a time like 201902 (partition for entries of
* February 2019) * February 2019)
*/ */
public ParititionId(final String partitionId) { public ParititionId(final String partitionId) {
super(); super();
this.partitionId = partitionId; this.partitionId = partitionId;
} }
public static ParititionId of(final String partitionId) { public static ParititionId of(final String partitionId) {
return new ParititionId(partitionId); return new ParititionId(partitionId);
} }
@Override @Override
public int compareTo(final ParititionId other) { public int compareTo(final ParititionId other) {
return partitionId.compareTo(other.getPartitionId()); return partitionId.compareTo(other.getPartitionId());
} }
/** /**
* @return the id, e.g. a time like 201902 (partition for entries of February * @return the id, e.g. a time like 201902 (partition for entries of February
* 2019) * 2019)
*/ */
public String getPartitionId() { public String getPartitionId() {
return partitionId; return partitionId;
} }
@Override @Override
public String toString() { public String toString() {
return partitionId; return partitionId;
} }
/* /*
* non-standard hashcode implementation! This class is just a wrapper for * non-standard hashcode implementation! This class is just a wrapper for
* string, so we delegate directly to String.hashCode(). * string, so we delegate directly to String.hashCode().
*/ */
@Override @Override
public int hashCode() { public int hashCode() {
return partitionId.hashCode(); return partitionId.hashCode();
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final ParititionId other = (ParititionId) obj; final ParititionId other = (ParititionId) obj;
if (partitionId == null) { if (partitionId == null) {
if (other.partitionId != null) if (other.partitionId != null)
return false; return false;
} else if (!partitionId.equals(other.partitionId)) } else if (!partitionId.equals(other.partitionId))
return false; return false;
return true; return true;
} }
} }

4
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareEncoderDecoder.java
View File

@@ -4,7 +4,7 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public interface PartitionAwareEncoderDecoder<V, P> extends EncoderDecoder<P> { public interface PartitionAwareEncoderDecoder<V, P> extends EncoderDecoder<P> {
public P encodeValue(V v); public P encodeValue(V v);
public V decodeValue(ParititionId partitionId, P p); public V decodeValue(ParititionId partitionId, P p);
} }

54
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareWrapper.java
View File

@@ -4,37 +4,37 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public final class PartitionAwareWrapper<O> implements PartitionAwareEncoderDecoder<O, O> { public final class PartitionAwareWrapper<O> implements PartitionAwareEncoderDecoder<O, O> {
private final EncoderDecoder<O> delegate; private final EncoderDecoder<O> delegate;
public PartitionAwareWrapper(final EncoderDecoder<O> delegate) { public PartitionAwareWrapper(final EncoderDecoder<O> delegate) {
this.delegate = delegate; this.delegate = delegate;
} }
@Override @Override
public byte[] encode(final O object) { public byte[] encode(final O object) {
return delegate.encode(object); return delegate.encode(object);
} }
@Override @Override
public O decode(final byte[] bytes) { public O decode(final byte[] bytes) {
return delegate.decode(bytes); return delegate.decode(bytes);
} }
@Override @Override
public O encodeValue(final O v) { public O encodeValue(final O v) {
return v; return v;
} }
@Override @Override
public O decodeValue(final ParititionId partitionId, final O p) { public O decodeValue(final ParititionId partitionId, final O p) {
return p; return p;
} }
public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) { public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) {
return new PartitionAwareWrapper<>(encoder); return new PartitionAwareWrapper<>(encoder);
} }
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return delegate.getEmptyValue(); return delegate.getEmptyValue();
} }
} }

108
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionDiskStore.java
View File

@@ -14,68 +14,68 @@ import org.lucares.pdb.blockstorage.LongStreamFile;
import org.lucares.pdb.diskstorage.DiskStorage; import org.lucares.pdb.diskstorage.DiskStorage;
public class PartitionDiskStore { public class PartitionDiskStore {
private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>(); private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>();
private final Function<ParititionId, DiskStorage> creator; private final Function<ParititionId, DiskStorage> creator;
private final Function<ParititionId, DiskStorage> supplier; private final Function<ParititionId, DiskStorage> supplier;
public PartitionDiskStore(final Path storageBasePath, final String filename) { public PartitionDiskStore(final Path storageBasePath, final String filename) {
creator = partitionId -> { creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename); final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
final boolean isNew = !Files.exists(file); final boolean isNew = !Files.exists(file);
final DiskStorage diskStorage = new DiskStorage(file, storageBasePath); final DiskStorage diskStorage = new DiskStorage(file, storageBasePath);
if (isNew) { if (isNew) {
diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE); diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE);
} }
return diskStorage; return diskStorage;
}; };
supplier = partitionId -> { supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename); final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) { if (Files.exists(file)) {
return new DiskStorage(file, storageBasePath); return new DiskStorage(file, storageBasePath);
} }
return null; return null;
}; };
} }
public DiskStorage getExisting(final ParititionId partitionId) { public DiskStorage getExisting(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, supplier); return diskStorages.computeIfAbsent(partitionId, supplier);
} }
public DiskStorage getCreateIfNotExists(final ParititionId partitionId) { public DiskStorage getCreateIfNotExists(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, creator); return diskStorages.computeIfAbsent(partitionId, creator);
} }
public long allocateBlock(final ParititionId partitionId, final int blockSize) { public long allocateBlock(final ParititionId partitionId, final int blockSize) {
final DiskStorage diskStorage = getCreateIfNotExists(partitionId); final DiskStorage diskStorage = getCreateIfNotExists(partitionId);
return diskStorage.allocateBlock(blockSize); return diskStorage.allocateBlock(blockSize);
} }
public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) { public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) {
try { try {
final DiskStorage diskStorage = getExisting(partitionId); final DiskStorage diskStorage = getExisting(partitionId);
return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage); return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage);
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
public void close() { public void close() {
final List<Throwable> throwables = new ArrayList<>(); final List<Throwable> throwables = new ArrayList<>();
for (final DiskStorage diskStorage : diskStorages.values()) { for (final DiskStorage diskStorage : diskStorages.values()) {
try { try {
diskStorage.close(); diskStorage.close();
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
throwables.add(e); throwables.add(e);
} }
} }
if (!throwables.isEmpty()) { if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException(); final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed); throwables.forEach(ex::addSuppressed);
throw ex; throw ex;
} }
} }
} }

2
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionIdSource.java
View File

@@ -3,5 +3,5 @@ package org.lucares.pdb.datastore.internal;
import java.util.Set; import java.util.Set;
public interface PartitionIdSource { public interface PartitionIdSource {
Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions); Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions);
} }

140
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionLongList.java
View File

@@ -9,87 +9,87 @@ import java.util.Set;
import org.lucares.collections.LongList; import org.lucares.collections.LongList;
public class PartitionLongList implements Iterable<ParititionId> { public class PartitionLongList implements Iterable<ParititionId> {
private final Map<ParititionId, LongList> lists = new HashMap<>(); private final Map<ParititionId, LongList> lists = new HashMap<>();
public LongList put(final ParititionId partitionId, final LongList longList) { public LongList put(final ParititionId partitionId, final LongList longList) {
return lists.put(partitionId, longList); return lists.put(partitionId, longList);
} }
public LongList get(final ParititionId partitionId) { public LongList get(final ParititionId partitionId) {
return lists.get(partitionId); return lists.get(partitionId);
} }
@Override @Override
public Iterator<ParititionId> iterator() { public Iterator<ParititionId> iterator() {
return lists.keySet().iterator(); return lists.keySet().iterator();
} }
public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) { public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList(); final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>(); final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet()); partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet()); partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) { for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId); final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId); final LongList y = b.get(partitionId);
if (x != null && y != null) { if (x != null && y != null) {
final LongList intersection = LongList.intersection(x, y); final LongList intersection = LongList.intersection(x, y);
result.put(partitionId, intersection); result.put(partitionId, intersection);
} else { } else {
// one list is empty => the intersection is empty // one list is empty => the intersection is empty
} }
} }
return result; return result;
} }
public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) { public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList(); final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>(); final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet()); partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet()); partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) { for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId); final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId); final LongList y = b.get(partitionId);
if (x != null && y != null) { if (x != null && y != null) {
final LongList intersection = LongList.union(x, y); final LongList intersection = LongList.union(x, y);
result.put(partitionId, intersection); result.put(partitionId, intersection);
} else if (x != null) { } else if (x != null) {
result.put(partitionId, x.clone()); result.put(partitionId, x.clone());
} else if (y != null) { } else if (y != null) {
result.put(partitionId, y.clone()); result.put(partitionId, y.clone());
} }
} }
return result; return result;
} }
public int size() { public int size() {
int size = 0; int size = 0;
for (final LongList longList : lists.values()) { for (final LongList longList : lists.values()) {
size += longList.size(); size += longList.size();
} }
return size; return size;
} }
public boolean isSorted() { public boolean isSorted() {
for (final LongList longList : lists.values()) { for (final LongList longList : lists.values()) {
if (!longList.isSorted()) { if (!longList.isSorted()) {
return false; return false;
} }
} }
return true; return true;
} }
public void removeAll(final PartitionLongList remove) { public void removeAll(final PartitionLongList remove) {
for (final ParititionId partitionId : lists.keySet()) { for (final ParititionId partitionId : lists.keySet()) {
final LongList removeLongList = remove.get(partitionId); final LongList removeLongList = remove.get(partitionId);
if (removeLongList != null) { if (removeLongList != null) {
lists.get(partitionId).removeAll(removeLongList); lists.get(partitionId).removeAll(removeLongList);
} }
} }
} }
} }

212
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java
View File

@@ -25,130 +25,130 @@ import org.lucares.pdb.map.Visitor;
*/ */
public class PartitionPersistentMap<K, V, P> implements AutoCloseable { public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>(); private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final Function<ParititionId, PersistentMap<K, P>> creator; private final Function<ParititionId, PersistentMap<K, P>> creator;
private final Function<ParititionId, PersistentMap<K, P>> supplier; private final Function<ParititionId, PersistentMap<K, P>> supplier;
private final PartitionAwareEncoderDecoder<V, P> valueEncoder; private final PartitionAwareEncoderDecoder<V, P> valueEncoder;
public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder, public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder,
final PartitionAwareEncoderDecoder<V, P> valueEncoder) { final PartitionAwareEncoderDecoder<V, P> valueEncoder) {
this.valueEncoder = valueEncoder; this.valueEncoder = valueEncoder;
creator = partitionId -> { creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename); final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder); return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
}; };
supplier = partitionId -> { supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename); final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) { if (Files.exists(file)) {
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder); return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
} }
return null; return null;
}; };
preload(storageBasePath); preload(storageBasePath);
} }
private void preload(final Path storageBasePath) { private void preload(final Path storageBasePath) {
try { try {
Files.list(storageBasePath)// Files.list(storageBasePath)//
.filter(Files::isDirectory)// .filter(Files::isDirectory)//
.map(Path::getFileName)// .map(Path::getFileName)//
.map(Path::toString)// .map(Path::toString)//
.map(ParititionId::of)// .map(ParititionId::of)//
.forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier)); .forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier));
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
private Set<ParititionId> getAllPartitionIds() { private Set<ParititionId> getAllPartitionIds() {
return maps.keySet(); return maps.keySet();
} }
public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) { public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) {
return partitionIdSource.toPartitionIds(getAllPartitionIds()); return partitionIdSource.toPartitionIds(getAllPartitionIds());
} }
private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) { private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, supplier); return maps.computeIfAbsent(partitionId, supplier);
} }
private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) { private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, creator); return maps.computeIfAbsent(partitionId, creator);
} }
public V getValue(final ParititionId partitionId, final K key) { public V getValue(final ParititionId partitionId, final K key) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId); final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
final P persistedValue = map != null ? map.getValue(key) : null; final P persistedValue = map != null ? map.getValue(key) : null;
return valueEncoder.decodeValue(partitionId, persistedValue); return valueEncoder.decodeValue(partitionId, persistedValue);
} }
public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) { public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) {
final List<V> result = new ArrayList<>(); final List<V> result = new ArrayList<>();
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds()); final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) { for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId); final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
if (map != null) { if (map != null) {
final V value = valueEncoder.decodeValue(partitionId, map.getValue(key)); final V value = valueEncoder.decodeValue(partitionId, map.getValue(key));
if (value != null) { if (value != null) {
result.add(value); result.add(value);
} }
} }
} }
return result; return result;
} }
public V putValue(final ParititionId partitionId, final K key, final V value) { public V putValue(final ParititionId partitionId, final K key, final V value) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId); final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
final P persistedValue = valueEncoder.encodeValue(value); final P persistedValue = valueEncoder.encodeValue(value);
final P previousPersistedValue = map.putValue(key, persistedValue); final P previousPersistedValue = map.putValue(key, persistedValue);
return valueEncoder.decodeValue(partitionId, previousPersistedValue); return valueEncoder.decodeValue(partitionId, previousPersistedValue);
} }
public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) { public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId); final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) { if (map != null) {
map.visitValues(keyPrefix, (k, p) -> { map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p); final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value); visitor.visit(k, value);
}); });
} }
} }
public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) { public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) {
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds()); final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) { for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId); final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) { if (map != null) {
map.visitValues(keyPrefix, (k, p) -> { map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p); final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value); visitor.visit(k, value);
}); });
} }
} }
} }
@Override @Override
public void close() { public void close() {
final List<Throwable> throwables = new ArrayList<>(); final List<Throwable> throwables = new ArrayList<>();
for (final PersistentMap<K, P> map : maps.values()) { for (final PersistentMap<K, P> map : maps.values()) {
try { try {
map.close(); map.close();
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
throwables.add(e); throwables.add(e);
} }
} }
if (!throwables.isEmpty()) { if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException(); final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed); throwables.forEach(ex::addSuppressed);
throw ex; throw ex;
} }
} }
} }

88
data-store/src/main/java/org/lucares/pdb/datastore/internal/PdbWriter.java
View File

@@ -17,62 +17,62 @@ import org.slf4j.LoggerFactory;
*/ */
class PdbWriter implements AutoCloseable, Flushable { class PdbWriter implements AutoCloseable, Flushable {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class); private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
private final PdbFile pdbFile; private final PdbFile pdbFile;
private long lastEpochMilli; private long lastEpochMilli;
private final TimeSeriesFile timeSeriesFile; private final TimeSeriesFile timeSeriesFile;
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) { public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) {
this.pdbFile = pdbFile; this.pdbFile = pdbFile;
timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage); timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct? final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct?
lastEpochMilli = optionalLastValue.orElse(0L); lastEpochMilli = optionalLastValue.orElse(0L);
} }
public PdbFile getPdbFile() { public PdbFile getPdbFile() {
return pdbFile; return pdbFile;
} }
public long getDateOffsetAsEpochMilli() { public long getDateOffsetAsEpochMilli() {
return lastEpochMilli; return lastEpochMilli;
} }
public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException { public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException {
try { try {
timeSeriesFile.appendTimeValue(epochMilli, value); timeSeriesFile.appendTimeValue(epochMilli, value);
lastEpochMilli = epochMilli; lastEpochMilli = epochMilli;
} catch (final RuntimeException e) { } catch (final RuntimeException e) {
throw new WriteException(e); throw new WriteException(e);
} }
} }
@Override @Override
public void close() { public void close() {
LOGGER.debug("close PdbWriter {}", pdbFile); LOGGER.debug("close PdbWriter {}", pdbFile);
timeSeriesFile.close(); timeSeriesFile.close();
} }
@Override @Override
public void flush() { public void flush() {
timeSeriesFile.flush(); timeSeriesFile.flush();
} }
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) { public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) { try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) { for (final Entry entry : entries) {
writer.write(entry.getEpochMilli(), entry.getValue()); writer.write(entry.getEpochMilli(), entry.getValue());
} }
} }
} }
@Override @Override
public String toString() { public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]"; return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";
} }
} }

506
data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java
View File

@@ -143,321 +143,321 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
* *
*/ */
public class QueryCompletionIndex implements AutoCloseable { public class QueryCompletionIndex implements AutoCloseable {
private static final class TwoTags { private static final class TwoTags {
private final Tag tagA; private final Tag tagA;
private final Tag tagB; private final Tag tagB;
public TwoTags(final Tag tagA, final Tag tagB) { public TwoTags(final Tag tagA, final Tag tagB) {
this.tagA = tagA; this.tagA = tagA;
this.tagB = tagB; this.tagB = tagB;
} }
public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) { public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) {
tagA = new Tag(fieldA, valueA); tagA = new Tag(fieldA, valueA);
tagB = new Tag(fieldB, valueB); tagB = new Tag(fieldB, valueB);
} }
public Tag getTagA() { public Tag getTagA() {
return tagA; return tagA;
} }
public Tag getTagB() { public Tag getTagB() {
return tagB; return tagB;
} }
@Override @Override
public String toString() { public String toString() {
return tagA + "::" + tagB; return tagA + "::" + tagB;
} }
} }
public static final class FieldField { public static final class FieldField {
private final int fieldA; private final int fieldA;
private final int fieldB; private final int fieldB;
public FieldField(final int fieldA, final int fieldB) { public FieldField(final int fieldA, final int fieldB) {
this.fieldA = fieldA; this.fieldA = fieldA;
this.fieldB = fieldB; this.fieldB = fieldB;
} }
public int getFieldA() { public int getFieldA() {
return fieldA; return fieldA;
} }
public int getFieldB() { public int getFieldB() {
return fieldB; return fieldB;
} }
@Override @Override
public String toString() { public String toString() {
return fieldA + "::" + fieldB; return fieldA + "::" + fieldB;
} }
} }
private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> { private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> {
@Override @Override
public byte[] encode(final TwoTags tagAndField) { public byte[] encode(final TwoTags tagAndField) {
final LongList tmp = new LongList(4); final LongList tmp = new LongList(4);
final Tag tagA = tagAndField.getTagA(); final Tag tagA = tagAndField.getTagA();
final Tag tagB = tagAndField.getTagB(); final Tag tagB = tagAndField.getTagB();
tmp.add(tagB.getKey()); tmp.add(tagB.getKey());
tmp.add(tagA.getKey()); tmp.add(tagA.getKey());
if (tagA.getValue() >= 0) { if (tagA.getValue() >= 0) {
tmp.add(tagA.getValue()); tmp.add(tagA.getValue());
// A query for tagA.key and tagA.value and tagB.key is done by setting // A query for tagA.key and tagA.value and tagB.key is done by setting
// tagB.value==-1. // tagB.value==-1.
// The query is then executed as a prefix search. Thus tagB.value must not be // The query is then executed as a prefix search. Thus tagB.value must not be
// part of the byte array that is returned. // part of the byte array that is returned.
if (tagB.getValue() >= 0) { if (tagB.getValue() >= 0) {
tmp.add(tagB.getValue()); tmp.add(tagB.getValue());
} }
} else { } else {
Preconditions.checkSmaller(tagB.getValue(), 0, Preconditions.checkSmaller(tagB.getValue(), 0,
"if no value for tagA is given, then tagB must also be empty"); "if no value for tagA is given, then tagB must also be empty");
} }
return VariableByteEncoder.encode(tmp); return VariableByteEncoder.encode(tmp);
} }
@Override @Override
public TwoTags decode(final byte[] bytes) { public TwoTags decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes); final LongList tmp = VariableByteEncoder.decode(bytes);
final int tagBKey = (int) tmp.get(0); final int tagBKey = (int) tmp.get(0);
final int tagAKey = (int) tmp.get(1); final int tagAKey = (int) tmp.get(1);
final int tagAValue = (int) tmp.get(2); final int tagAValue = (int) tmp.get(2);
final int tagBValue = (int) tmp.get(3); final int tagBValue = (int) tmp.get(3);
final Tag tagA = new Tag(tagAKey, tagAValue); final Tag tagA = new Tag(tagAKey, tagAValue);
final Tag tagB = new Tag(tagBKey, tagBValue); final Tag tagB = new Tag(tagBKey, tagBValue);
return new TwoTags(tagA, tagB); return new TwoTags(tagA, tagB);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0, 0, 0, 0 }; return new byte[] { 0, 0, 0, 0 };
} }
} }
private static final class EncoderTag implements EncoderDecoder<Tag> { private static final class EncoderTag implements EncoderDecoder<Tag> {
@Override @Override
public byte[] encode(final Tag tag) { public byte[] encode(final Tag tag) {
final LongList longList = new LongList(2); final LongList longList = new LongList(2);
longList.add(tag.getKey()); longList.add(tag.getKey());
if (tag.getValue() >= 0) { if (tag.getValue() >= 0) {
longList.add(tag.getValue()); longList.add(tag.getValue());
} }
return VariableByteEncoder.encode(longList); return VariableByteEncoder.encode(longList);
} }
@Override @Override
public Tag decode(final byte[] bytes) { public Tag decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes); final LongList tmp = VariableByteEncoder.decode(bytes);
final int key = (int) tmp.get(0); final int key = (int) tmp.get(0);
final int value = (int) tmp.get(1); final int value = (int) tmp.get(1);
return new Tag(key, value); return new Tag(key, value);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0 }; return new byte[] { 0 };
} }
} }
private static final class EncoderField implements EncoderDecoder<String> { private static final class EncoderField implements EncoderDecoder<String> {
@Override @Override
public byte[] encode(final String field) { public byte[] encode(final String field) {
if (field.isEmpty()) { if (field.isEmpty()) {
return new byte[0]; return new byte[0];
} }
return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field)); return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field));
} }
@Override @Override
public String decode(final byte[] bytes) { public String decode(final byte[] bytes) {
final long compressedString = VariableByteEncoder.decodeFirstValue(bytes); final long compressedString = VariableByteEncoder.decodeFirstValue(bytes);
return Tags.STRING_COMPRESSOR.get((int) compressedString); return Tags.STRING_COMPRESSOR.get((int) compressedString);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] { 0 }; return new byte[] { 0 };
} }
} }
private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex; private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex;
private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex; private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex;
private final PartitionPersistentMap<String, Empty, Empty> fieldIndex; private final PartitionPersistentMap<String, Empty, Empty> fieldIndex;
public QueryCompletionIndex(final Path basePath) throws IOException { public QueryCompletionIndex(final Path basePath) throws IOException {
tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(), tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER)); PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs", fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs",
new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER)); new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(), fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER)); PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
} }
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException { public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final List<Tag> listOfTagsA = tags.toTags(); final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags(); final List<Tag> listOfTagsB = tags.toTags();
// index all combinations of tagA and tagB and fieldA to fieldB // index all combinations of tagA and tagB and fieldA to fieldB
for (final Tag tagA : listOfTagsA) { for (final Tag tagA : listOfTagsA) {
for (final Tag tagB : listOfTagsB) { for (final Tag tagB : listOfTagsB) {
final TwoTags key = new TwoTags(tagA, tagB); final TwoTags key = new TwoTags(tagA, tagB);
tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE); tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE);
} }
} }
// create indices of all tags and all fields // create indices of all tags and all fields
for (final Tag tag : listOfTagsA) { for (final Tag tag : listOfTagsA) {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE); fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE); fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE);
} }
} }
@Override @Override
public void close() throws IOException { public void close() throws IOException {
tagToTagIndex.close(); tagToTagIndex.close();
} }
/** /**
* Find values for fieldB that are yield results when executing the query * Find values for fieldB that are yield results when executing the query
* "fieldA=valueA and fieldB=???" * "fieldA=valueA and fieldB=???"
* *
* @param dateRange the date range * @param dateRange the date range
* @param fieldA the other field of the and expression * @param fieldA the other field of the and expression
* @param valueA {@link GlobMatcher} for the value of the other field * @param valueA {@link GlobMatcher} for the value of the other field
* @param fieldB the field we are searching values for * @param fieldB the field we are searching values for
* @return values of fieldB * @return values of fieldB
*/ */
public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA, public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA,
final String fieldB) { final String fieldB) {
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null); final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> { tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final String vA = k.getTagA().getValueAsString(); final String vA = k.getTagA().getValueAsString();
if (valueA.matches(vA)) { if (valueA.matches(vA)) {
result.add(k.getTagB().getValueAsString()); result.add(k.getTagB().getValueAsString());
} }
}); });
return result; return result;
} }
/** /**
* Find values for fieldB that are yield results when executing the query * Find values for fieldB that are yield results when executing the query
* "tag.field=tag.value and fieldB=???" * "tag.field=tag.value and fieldB=???"
* *
* @param dateRange the date range * @param dateRange the date range
* @param tag the other tag * @param tag the other tag
* @param field the field we are searching values for * @param field the field we are searching values for
* @return values for the field * @return values for the field
*/ */
public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) { public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) {
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
final int tagBKey = Tags.STRING_COMPRESSOR.put(field); final int tagBKey = Tags.STRING_COMPRESSOR.put(field);
final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See
// EncoderTwoTags // EncoderTwoTags
final TwoTags keyPrefix = new TwoTags(tag, tagB); final TwoTags keyPrefix = new TwoTags(tag, tagB);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> { tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getTagB().getValueAsString()); result.add(k.getTagB().getValueAsString());
}); });
return result; return result;
} }
/** /**
* Find all values for the given field. * Find all values for the given field.
* *
* @param dateRange the date range * @param dateRange the date range
* @param field the field * @param field the field
* @return the values * @return the values
*/ */
public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) { public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) {
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
final int tagKey = Tags.STRING_COMPRESSOR.put(field); final int tagKey = Tags.STRING_COMPRESSOR.put(field);
final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> { fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getValueAsString()); result.add(k.getValueAsString());
}); });
return result; return result;
} }
/** /**
* Find values for {@code field} that will yield results for the query * Find values for {@code field} that will yield results for the query
* "tag.field=tag.value and not field=???". * "tag.field=tag.value and not field=???".
* <p> * <p>
* *
* @param dateRange the date range * @param dateRange the date range
* @param tag the other tag * @param tag the other tag
* @param field the field we are searching values for * @param field the field we are searching values for
* @return the values * @return the values
*/ */
public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag, public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag,
final String field) { final String field) {
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null); final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null);
final int negatedValueA = tag.getValue(); final int negatedValueA = tag.getValue();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> { tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final int valueA = k.getTagA().getValue(); final int valueA = k.getTagA().getValue();
if (valueA != negatedValueA) { if (valueA != negatedValueA) {
result.add(k.getTagB().getValueAsString()); result.add(k.getTagB().getValueAsString());
} }
}); });
return result; return result;
} }
public SortedSet<String> findAllFields(final DateTimeRange dateRange) { public SortedSet<String> findAllFields(final DateTimeRange dateRange) {
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> { fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
result.add(k); result.add(k);
}); });
return result; return result;
} }
public boolean hasField(final DateTimeRange dateRange, final String field) { public boolean hasField(final DateTimeRange dateRange, final String field) {
final AtomicBoolean found = new AtomicBoolean(false); final AtomicBoolean found = new AtomicBoolean(false);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange); final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> { fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
if (k.equals(field)) { if (k.equals(field)) {
found.set(true); found.set(true);
} }
}); });
return found.get(); return found.get();
} }
} }

92
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagEncoderDecoder.java
View File

@@ -7,58 +7,58 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
import org.lucares.utils.byteencoder.VariableByteEncoder; import org.lucares.utils.byteencoder.VariableByteEncoder;
class TagEncoderDecoder implements EncoderDecoder<Tag> { class TagEncoderDecoder implements EncoderDecoder<Tag> {
@Override @Override
public byte[] encode(final Tag tag) { public byte[] encode(final Tag tag) {
final LongList keyAndValueCompressed = new LongList(2); final LongList keyAndValueCompressed = new LongList(2);
final String key = tag.getKeyAsString(); final String key = tag.getKeyAsString();
final byte[] result; final byte[] result;
if (!key.isEmpty()) { if (!key.isEmpty()) {
final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key); final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key);
keyAndValueCompressed.add(keyAsLong); keyAndValueCompressed.add(keyAsLong);
final String value = tag.getValueAsString(); final String value = tag.getValueAsString();
if (!value.isEmpty()) { if (!value.isEmpty()) {
final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value); final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value);
keyAndValueCompressed.add(valueAsLong); keyAndValueCompressed.add(valueAsLong);
} }
result = VariableByteEncoder.encode(keyAndValueCompressed); result = VariableByteEncoder.encode(keyAndValueCompressed);
} else { } else {
result = new byte[0]; result = new byte[0];
} }
return result; return result;
} }
@Override @Override
public Tag decode(final byte[] bytes) { public Tag decode(final byte[] bytes) {
final LongList compressedStrings = VariableByteEncoder.decode(bytes); final LongList compressedStrings = VariableByteEncoder.decode(bytes);
final Tag result; final Tag result;
switch (compressedStrings.size()) { switch (compressedStrings.size()) {
case 0: case 0:
result = new Tag("", ""); result = new Tag("", "");
break; break;
case 1: case 1:
final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0)); final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
result = new Tag(k, ""); result = new Tag(k, "");
break; break;
case 2: case 2:
final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0)); final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1)); final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1));
result = new Tag(key, value); result = new Tag(key, value);
break; break;
default: default:
throw new IllegalStateException("too many values: " + compressedStrings); throw new IllegalStateException("too many values: " + compressedStrings);
} }
return result; return result;
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] {0}; return new byte[] { 0 };
} }
} }

26
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagsEncoderDecoder.java
View File

@@ -4,18 +4,18 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.map.PersistentMap.EncoderDecoder; import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
class TagsEncoderDecoder implements EncoderDecoder<Tags> { class TagsEncoderDecoder implements EncoderDecoder<Tags> {
@Override @Override
public byte[] encode(final Tags tags) { public byte[] encode(final Tags tags) {
return tags.toBytes(); return tags.toBytes();
} }
@Override @Override
public Tags decode(final byte[] bytes) { public Tags decode(final byte[] bytes) {
return Tags.fromBytes(bytes); return Tags.fromBytes(bytes);
} }
@Override @Override
public byte[] getEmptyValue() { public byte[] getEmptyValue() {
return new byte[] {}; return new byte[] {};
} }
} }

64
data-store/src/main/java/org/lucares/pdb/datastore/lang/CandidateGrouper.java
View File

@@ -9,43 +9,43 @@ import java.util.TreeSet;
import java.util.regex.Pattern; import java.util.regex.Pattern;
public class CandidateGrouper { public class CandidateGrouper {
public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) { public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) {
final TreeSet<String> result = new TreeSet<>(); final TreeSet<String> result = new TreeSet<>();
final int numDotsInValue = countDotsInValue(queryWithCaretMarker); final int numDotsInValue = countDotsInValue(queryWithCaretMarker);
for (final String value : values) { for (final String value : values) {
// keep everything up to the (numDotsInValue+1)-th // keep everything up to the (numDotsInValue+1)-th
final String[] token = value.split(Pattern.quote(".")); final String[] token = value.split(Pattern.quote("."));
final List<String> tokenlist = new ArrayList<>(Arrays.asList(token)); final List<String> tokenlist = new ArrayList<>(Arrays.asList(token));
final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1); final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1);
String shortenedValue = String.join(".", prefix); String shortenedValue = String.join(".", prefix);
if (tokenlist.size() > numDotsInValue + 1) { if (tokenlist.size() > numDotsInValue + 1) {
shortenedValue += "."; shortenedValue += ".";
} }
result.add(shortenedValue); result.add(shortenedValue);
} }
return result; return result;
} }
private int countDotsInValue(final String queryWithCaretMarker) { private int countDotsInValue(final String queryWithCaretMarker) {
int count = 0; int count = 0;
int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1; int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1;
final String delimiter = " (),=!"; final String delimiter = " (),=!";
while (index >= 0) { while (index >= 0) {
final char c = queryWithCaretMarker.charAt(index); final char c = queryWithCaretMarker.charAt(index);
if (delimiter.indexOf(c) >= 0) { if (delimiter.indexOf(c) >= 0) {
break; break;
} }
if (c == '.') { if (c == '.') {
count++; count++;
} }
index--; index--;
} }
return count; return count;
} }
} }

18
data-store/src/main/java/org/lucares/pdb/datastore/lang/ErrorListener.java
View File

@@ -6,14 +6,14 @@ import org.antlr.v4.runtime.Recognizer;
public class ErrorListener extends BaseErrorListener { public class ErrorListener extends BaseErrorListener {
@Override @Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) { final int charPositionInLine, final String msg, final RecognitionException e) {
final int lineStart = line; final int lineStart = line;
final int startIndex = charPositionInLine; final int startIndex = charPositionInLine;
final int lineStop = line; final int lineStop = line;
final int stopIndex = charPositionInLine; final int stopIndex = charPositionInLine;
throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex); throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex);
} }
} }

1426
data-store/src/main/java/org/lucares/pdb/datastore/lang/Expression.java
View File

File diff suppressed because it is too large Load Diff

263
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java
View File

@@ -26,179 +26,180 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class ExpressionToDocIdVisitor extends ExpressionVisitor<PartitionLongList> { public class ExpressionToDocIdVisitor extends ExpressionVisitor<PartitionLongList> {
private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class); private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class);
private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId; private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId;
private final PartitionDiskStore diskStorage; private final PartitionDiskStore diskStorage;
private final DatePartitioner datePartitioner; private final DatePartitioner datePartitioner;
public ExpressionToDocIdVisitor(final DateTimeRange dateRange, public ExpressionToDocIdVisitor(final DateTimeRange dateRange,
final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) { final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) {
this.datePartitioner = new DatePartitioner(dateRange); this.datePartitioner = new DatePartitioner(dateRange);
this.keyToValueToDocId = keyToValueToDocsId; this.keyToValueToDocId = keyToValueToDocsId;
this.diskStorage = diskStorage; this.diskStorage = diskStorage;
} }
@Override @Override
public PartitionLongList visit(final And expression) { public PartitionLongList visit(final And expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this); final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this); final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime(); final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles); final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles);
LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0, LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size()); result.size());
assert result.isSorted(); assert result.isSorted();
return result; return result;
} }
@Override @Override
public PartitionLongList visit(final Or expression) { public PartitionLongList visit(final Or expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this); final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this); final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime(); final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles); final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles);
LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0, LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size()); result.size());
assert result.isSorted(); assert result.isSorted();
return result; return result;
} }
@Override @Override
public PartitionLongList visit(final Not expression) { public PartitionLongList visit(final Not expression) {
final Expression negatedExpression = expression.getExpression(); final Expression negatedExpression = expression.getExpression();
final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this); final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this);
final long start = System.nanoTime(); final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds(); final PartitionLongList result = getAllDocIds();
result.removeAll(docIdsToBeNegated); result.removeAll(docIdsToBeNegated);
LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0, LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size()); result.size());
return result; return result;
} }
@Override @Override
public PartitionLongList visit(final Parentheses parentheses) { public PartitionLongList visit(final Parentheses parentheses) {
throw new UnsupportedOperationException( throw new UnsupportedOperationException(
"Parenthesis not supported. The correct order should come from the parser."); "Parenthesis not supported. The correct order should come from the parser.");
} }
@Override @Override
public PartitionLongList visit(final Expression.MatchAll expression) { public PartitionLongList visit(final Expression.MatchAll expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds(); final PartitionLongList result = getAllDocIds();
LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0, LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size()); result.size());
return result; return result;
} }
@Override @Override
public PartitionLongList visit(final Expression.InExpression expression) { public PartitionLongList visit(final Expression.InExpression expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
final String propertyName = expression.getProperty(); final String propertyName = expression.getProperty();
final List<String> values = expression.getValues(); final List<String> values = expression.getValues();
PartitionLongList result = new PartitionLongList(); PartitionLongList result = new PartitionLongList();
for (final String value : values) { for (final String value : values) {
final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value)); final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value));
result = PartitionLongList.union(result, docIds); result = PartitionLongList.union(result, docIds);
} }
LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0, LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size()); result.size());
return result; return result;
} }
private PartitionLongList getAllDocIds() { private PartitionLongList getAllDocIds() {
final PartitionLongList result = new PartitionLongList(); final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner); final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) { for (final ParititionId partitionId : availablePartitionIds) {
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS); final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
if (blockOffset != null) { if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId); final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);
final LongList tmp = bsFile.asLongList(); final LongList tmp = bsFile.asLongList();
result.put(partitionId, tmp); result.put(partitionId, tmp);
} }
} }
return result; return result;
} }
private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) { private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) {
final PartitionLongList result = new PartitionLongList(); final PartitionLongList result = new PartitionLongList();
final long start = System.nanoTime(); final long start = System.nanoTime();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner); final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) { for (final ParititionId partitionId : availablePartitionIds) {
final List<LongList> docIdsForPartition = new ArrayList<>(); final List<LongList> docIdsForPartition = new ArrayList<>();
keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> { keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> {
if (valuePattern.matcher(tags.getValueAsString()).matches()) { if (valuePattern.matcher(tags.getValueAsString()).matches()) {
try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds, partitionId)) { try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds,
partitionId)) {
// We know that all LongLists coming from a BSFile are sorted, non-overlapping // We know that all LongLists coming from a BSFile are sorted, non-overlapping
// and increasing, that means we can just concatenate them and get a sorted // and increasing, that means we can just concatenate them and get a sorted
// list. // list.
final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList()); final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList());
final LongList concatenatedLists = concatenateLists(longLists); final LongList concatenatedLists = concatenateLists(longLists);
Preconditions.checkTrue(concatenatedLists.isSorted(), Preconditions.checkTrue(concatenatedLists.isSorted(),
"The LongLists containing document ids must be sorted, " "The LongLists containing document ids must be sorted, "
+ "non-overlapping and increasing, so that the concatenation " + "non-overlapping and increasing, so that the concatenation "
+ "is sorted. This is guaranteed by the fact that document ids " + "is sorted. This is guaranteed by the fact that document ids "
+ "are generated in monotonically increasing order."); + "are generated in monotonically increasing order.");
docIdsForPartition.add(concatenatedLists); docIdsForPartition.add(concatenatedLists);
} }
} }
}); });
final LongList mergedDocsIdsForPartition = merge(docIdsForPartition); final LongList mergedDocsIdsForPartition = merge(docIdsForPartition);
result.put(partitionId, mergedDocsIdsForPartition); result.put(partitionId, mergedDocsIdsForPartition);
} }
LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0); LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0);
return result; return result;
} }
private LongList merge(final Collection<LongList> lists) { private LongList merge(final Collection<LongList> lists) {
LongList result = new LongList(); LongList result = new LongList();
for (final LongList list : lists) { for (final LongList list : lists) {
result = LongList.union(result, list); result = LongList.union(result, list);
} }
return result; return result;
} }
private static LongList concatenateLists(final Collection<LongList> lists) { private static LongList concatenateLists(final Collection<LongList> lists) {
final int totalSize = lists.stream().mapToInt(LongList::size).sum(); final int totalSize = lists.stream().mapToInt(LongList::size).sum();
final LongList result = new LongList(totalSize); final LongList result = new LongList(totalSize);
for (final LongList list : lists) { for (final LongList list : lists) {
result.addAll(list); result.addAll(list);
} }
return result; return result;
} }
} }

66
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionVisitor.java
View File

@@ -1,47 +1,47 @@
package org.lucares.pdb.datastore.lang; package org.lucares.pdb.datastore.lang;
public abstract class ExpressionVisitor<T> { public abstract class ExpressionVisitor<T> {
public T visit(final Expression.And expression) { public T visit(final Expression.And expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.Or expression) { public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.Not expression) { public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.Property expression) { public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.Terminal expression) { public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.MatchAll expression) { public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.InExpression expression) { public T visit(final Expression.InExpression expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.Parentheses parentheses) { public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.AndCaretExpression expression) { public T visit(final Expression.AndCaretExpression expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.AndNotCaretExpression expression) { public T visit(final Expression.AndNotCaretExpression expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
public T visit(final Expression.CaretAndExpression expression) { public T visit(final Expression.CaretAndExpression expression) {
throw new UnsupportedOperationException(); throw new UnsupportedOperationException();
} }
} }

436
data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java
View File

@@ -22,278 +22,278 @@ import org.slf4j.LoggerFactory;
public class FindValuesForQueryCompletion extends ExpressionVisitor<SortedSet<String>> { public class FindValuesForQueryCompletion extends ExpressionVisitor<SortedSet<String>> {
private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret"); .getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret");
private static final Logger METRIC_LOGGER = LoggerFactory private static final Logger METRIC_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation"); .getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation");
private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> { private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> {
private final QueryCompletionIndex index; private final QueryCompletionIndex index;
private final String field; private final String field;
private final DateTimeRange dateTimeRange; private final DateTimeRange dateTimeRange;
public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange, public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange,
final QueryCompletionIndex queryCompletionIndex, final String field) { final QueryCompletionIndex queryCompletionIndex, final String field) {
this.dateTimeRange = dateTimeRange; this.dateTimeRange = dateTimeRange;
index = queryCompletionIndex; index = queryCompletionIndex;
this.field = field; this.field = field;
} }
@Override @Override
public SortedSet<String> visit(final Property property) { public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime(); final long start = System.nanoTime();
final SortedSet<String> result; final SortedSet<String> result;
final String fieldA = property.getField(); final String fieldA = property.getField();
final String valueA = property.getValue().getValue(); final String valueA = property.getValue().getValue();
final boolean hasField = index.hasField(dateTimeRange, fieldA); final boolean hasField = index.hasField(dateTimeRange, fieldA);
if (hasField) { if (hasField) {
final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA); final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA);
final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new); final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new);
final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size(); final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size();
final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was
// chosen // chosen
// arbitrarily // arbitrarily
if (useMultiFetch) { if (useMultiFetch) {
result = new TreeSet<>(); result = new TreeSet<>();
for (final String v : valuesA) { for (final String v : valuesA) {
final Tag tagA = new Tag(fieldA, v); final Tag tagA = new Tag(fieldA, v);
final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field); final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field);
result.addAll(tmp); result.addAll(tmp);
} }
} else { } else {
result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field); result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field);
} }
METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)", METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)",
useMultiFetch ? "multi-fetch" : "single-fetch", property, field, useMultiFetch ? "multi-fetch" : "single-fetch", property, field,
(System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100); (System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100);
} else { } else {
result = new TreeSet<>(); result = new TreeSet<>();
} }
return result; return result;
} }
@Override @Override
public SortedSet<String> visit(final InExpression expression) { public SortedSet<String> visit(final InExpression expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
final SortedSet<String> result; final SortedSet<String> result;
final String fieldA = expression.getProperty(); final String fieldA = expression.getProperty();
final List<String> values = expression.getValues(); final List<String> values = expression.getValues();
result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field); result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result; return result;
} }
@Override @Override
public SortedSet<String> visit(final And expression) { public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
try { try {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
if (left instanceof Property && right instanceof Not) { if (left instanceof Property && right instanceof Not) {
final Property leftProperty = (Property) left; final Property leftProperty = (Property) left;
final SortedSet<String> allValuesForField = leftProperty.visit(this); final SortedSet<String> allValuesForField = leftProperty.visit(this);
final Expression rightInnerExpression = ((Not) right).getExpression(); final Expression rightInnerExpression = ((Not) right).getExpression();
final SortedSet<String> rightResult = rightInnerExpression.visit(this); final SortedSet<String> rightResult = rightInnerExpression.visit(this);
return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new); return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new);
} else { } else {
final SortedSet<String> result = left.visit(this); final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this); final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult); result.retainAll(rightResult);
return result; return result;
} }
} finally { } finally {
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
} }
} }
@Override @Override
public SortedSet<String> visit(final Or expression) { public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this); final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this); final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult); result.addAll(rightResult);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result; return result;
} }
@Override @Override
public SortedSet<String> visit(final Not expression) { public SortedSet<String> visit(final Not expression) {
final long start = System.nanoTime(); final long start = System.nanoTime();
if (!(expression.getExpression() instanceof Property)) { if (!(expression.getExpression() instanceof Property)) {
throw new UnsupportedOperationException("NOT expressions like '" + expression throw new UnsupportedOperationException("NOT expressions like '" + expression
+ "' are not supported. Only 'NOT property=value' expressions are supported."); + "' are not supported. Only 'NOT property=value' expressions are supported.");
} }
final Property property = (Property) expression.getExpression(); final Property property = (Property) expression.getExpression();
final Tag tag = new Tag(property.getField(), property.getValueAsString()); final Tag tag = new Tag(property.getField(), property.getValueAsString());
final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field); final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field);
final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field); final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field);
final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField, final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField,
valuesNotForField, TreeSet::new); valuesNotForField, TreeSet::new);
final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField, final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField,
TreeSet::new); TreeSet::new);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result; return result;
} }
}
private final QueryCompletionIndex queryCompletionIndex;
private final DateTimeRange dateRange;
public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final QueryCompletionIndex queryCompletionIndex) {
this.dateRange = dateRange;
this.queryCompletionIndex = queryCompletionIndex;
}
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final String field = property.getField();
final String value = property.getValue().getValue();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valuePrefix;
if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
} }
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new); private final QueryCompletionIndex queryCompletionIndex;
METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override private final DateTimeRange dateRange;
public SortedSet<String> visit(final AndCaretExpression expression) {
final long start = System.nanoTime(); public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final Property caretExpression = expression.getCaretExpression(); final QueryCompletionIndex queryCompletionIndex) {
final String field = caretExpression.getField(); this.dateRange = dateRange;
final String valueWithCaretMarker = caretExpression.getValue().getValue(); this.queryCompletionIndex = queryCompletionIndex;
final String valuePrefix = valueWithCaretMarker.substring(0, }
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final Expression rightHandExpression = expression.getExpression(); @Override
public SortedSet<String> visit(final Property property) {
final SortedSet<String> candidateValues = rightHandExpression final long start = System.nanoTime();
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field)); final String field = property.getField();
final String value = property.getValue().getValue();
final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new); final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); final String valuePrefix;
return result;
}
@Override if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
public SortedSet<String> visit(final AndNotCaretExpression expression) { valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
}
final long start = System.nanoTime(); final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new);
final Property caretExpression = expression.getCaretExpression(); METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
final String field = caretExpression.getField(); return result;
final String valueWithCaretMarker = caretExpression.getValue().getValue(); }
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, @Override
caretExpression.getField()); public SortedSet<String> visit(final AndCaretExpression expression) {
final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
valuePattern, TreeSet::new);
final Expression rightHandExpression = expression.getExpression(); final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final SortedSet<String> rightHandValues = rightHandExpression final Expression rightHandExpression = expression.getExpression();
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
if (rightHandValues.size() == 1) { final SortedSet<String> candidateValues = rightHandExpression
// there is only one alternative and that one must not be chosen .visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
return Collections.emptySortedSet();
}
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new);
public SortedSet<String> visit(final Not expression) {
final String field; METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
final Expression innerExpression = expression.getExpression(); return result;
if (innerExpression instanceof Property) { }
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
@Override @Override
public SortedSet<String> visit(final Or expression) { public SortedSet<String> visit(final AndNotCaretExpression expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this); final long start = System.nanoTime();
final SortedSet<String> rightResult = right.visit(this); final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
result.addAll(rightResult); final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange,
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); caretExpression.getField());
return result; final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
} valuePattern, TreeSet::new);
@Override final Expression rightHandExpression = expression.getExpression();
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this); final SortedSet<String> rightHandValues = rightHandExpression
final SortedSet<String> rightResult = right.visit(this); .visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
result.retainAll(rightResult); if (rightHandValues.size() == 1) {
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); // there is only one alternative and that one must not be chosen
return result; return Collections.emptySortedSet();
} }
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Not expression) {
final String field;
final Expression innerExpression = expression.getExpression();
if (innerExpression instanceof Property) {
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
} }

94
data-store/src/main/java/org/lucares/pdb/datastore/lang/GloblikePattern.java
View File

@@ -12,70 +12,70 @@ import org.slf4j.LoggerFactory;
public class GloblikePattern { public class GloblikePattern {
private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class); private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class);
enum FilterMode { enum FilterMode {
KEEP_EQUAL KEEP_EQUAL
} }
public static Pattern globlikeToRegex(final String globlike) { public static Pattern globlikeToRegex(final String globlike) {
final String valueRegex = "^" + globlikeToPattern(globlike); final String valueRegex = "^" + globlikeToPattern(globlike);
LOGGER.trace(">{}< -> >{}<", globlike, valueRegex); LOGGER.trace(">{}< -> >{}<", globlike, valueRegex);
return Pattern.compile(valueRegex); return Pattern.compile(valueRegex);
} }
public static Pattern globlikeToRegex(final Iterable<String> globlikes) { public static Pattern globlikeToRegex(final Iterable<String> globlikes) {
final List<String> regex = new ArrayList<>(); final List<String> regex = new ArrayList<>();
for (final String globlike : globlikes) { for (final String globlike : globlikes) {
regex.add(globlikeToPattern(globlike)); regex.add(globlikeToPattern(globlike));
} }
final StringBuilder fullRegex = new StringBuilder("^("); final StringBuilder fullRegex = new StringBuilder("^(");
fullRegex.append(String.join("|", regex)); fullRegex.append(String.join("|", regex));
fullRegex.append(")"); fullRegex.append(")");
LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex); LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex);
return Pattern.compile(fullRegex.toString()); return Pattern.compile(fullRegex.toString());
} }
private static String globlikeToPattern(final String globlike) { private static String globlikeToPattern(final String globlike) {
// a character that cannot be in the globPattern // a character that cannot be in the globPattern
final String dotPlaceholder = "\ue003"; // fourth character in the private use area final String dotPlaceholder = "\ue003"; // fourth character in the private use area
final String valueRegex = globlike// final String valueRegex = globlike//
.replace("-", Pattern.quote("-"))// .replace("-", Pattern.quote("-"))//
.replace(".", dotPlaceholder)// .replace(".", dotPlaceholder)//
.replace("*", ".*")// .replace("*", ".*")//
.replace(dotPlaceholder, ".*\\.")// .replace(dotPlaceholder, ".*\\.")//
.replaceAll("([A-Z])", "[a-z]*$1"); .replaceAll("([A-Z])", "[a-z]*$1");
return valueRegex; return valueRegex;
} }
public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues, public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues,
final String valuePattern, final Supplier<T> generator) { final String valuePattern, final Supplier<T> generator) {
final T result = generator.get(); final T result = generator.get();
return filterValues(result, availableValues, valuePattern); return filterValues(result, availableValues, valuePattern);
} }
public static <T extends Collection<String>> T filterValues(final T result, public static <T extends Collection<String>> T filterValues(final T result,
final Collection<String> availableValues, final String valuePattern) { final Collection<String> availableValues, final String valuePattern) {
final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern); final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern);
for (final String value : availableValues) { for (final String value : availableValues) {
final Matcher matcher = pattern.matcher(value); final Matcher matcher = pattern.matcher(value);
if (matcher.find()) { if (matcher.find()) {
result.add(value); result.add(value);
} }
} }
return result; return result;
} }
} }

96
data-store/src/main/java/org/lucares/pdb/datastore/lang/IdentityExpressionVisitor.java
View File

@@ -14,66 +14,66 @@ import org.lucares.pdb.datastore.lang.Expression.Property;
* as base class for visitors that modify expressions. * as base class for visitors that modify expressions.
*/ */
public abstract class IdentityExpressionVisitor extends ExpressionVisitor<Expression> { public abstract class IdentityExpressionVisitor extends ExpressionVisitor<Expression> {
@Override @Override
public Expression visit(final And expression) { public Expression visit(final And expression) {
final Expression left = expression.getLeft().visit(this); final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this); final Expression right = expression.getRight().visit(this);
return new And(left, right); return new And(left, right);
} }
@Override @Override
public Expression visit(final Or expression) { public Expression visit(final Or expression) {
final Expression left = expression.getLeft().visit(this); final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this); final Expression right = expression.getRight().visit(this);
return new Or(left, right); return new Or(left, right);
} }
@Override @Override
public Expression visit(final Not expression) { public Expression visit(final Not expression) {
return new Not(expression.getExpression().visit(this)); return new Not(expression.getExpression().visit(this));
} }
@Override @Override
public Expression visit(final Property expression) { public Expression visit(final Property expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final Expression.Terminal expression) { public Expression visit(final Expression.Terminal expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final Expression.MatchAll expression) { public Expression visit(final Expression.MatchAll expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final Expression.InExpression expression) { public Expression visit(final Expression.InExpression expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final Parentheses parentheses) { public Expression visit(final Parentheses parentheses) {
return new Parentheses(parentheses.getExpression().visit(this)); return new Parentheses(parentheses.getExpression().visit(this));
} }
@Override @Override
public Expression visit(final AndCaretExpression expression) { public Expression visit(final AndCaretExpression expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final AndNotCaretExpression expression) { public Expression visit(final AndNotCaretExpression expression) {
return expression; return expression;
} }
@Override @Override
public Expression visit(final CaretAndExpression expression) { public Expression visit(final CaretAndExpression expression) {
return expression; return expression;
} }
} }

316
data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java
View File

@@ -21,203 +21,203 @@ import org.slf4j.LoggerFactory;
public class NewProposerParser implements QueryConstants { public class NewProposerParser implements QueryConstants {
private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class); private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class);
private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose"); private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose");
/* /*
* Regex matching a java identifier without a caret marker. We define it as a * Regex matching a java identifier without a caret marker. We define it as a
* blacklist, because this is easer. The regex is only used <em>after</em> the * blacklist, because this is easer. The regex is only used <em>after</em> the
* query has already been validated with the proper grammar. * query has already been validated with the proper grammar.
*/ */
private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*"; private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*";
private final QueryCompletionIndex queryCompletionIndex; private final QueryCompletionIndex queryCompletionIndex;
public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) { public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) {
this.queryCompletionIndex = queryCompletionIndex; this.queryCompletionIndex = queryCompletionIndex;
} }
public List<Proposal> propose(final QueryWithCaretMarker query) { public List<Proposal> propose(final QueryWithCaretMarker query) {
final long start = System.nanoTime(); final long start = System.nanoTime();
List<Proposal> proposals; List<Proposal> proposals;
if (StringUtils.isBlank(query.getQuery())) { if (StringUtils.isBlank(query.getQuery())) {
proposals = proposeForAllKeys(query.getDateRange()); proposals = proposeForAllKeys(query.getDateRange());
} else { } else {
final List<Proposal> foundProposals = proposalsForValues(query); final List<Proposal> foundProposals = proposalsForValues(query);
if (foundProposals.isEmpty()) { if (foundProposals.isEmpty()) {
proposals = proposalsForNonValues(query); proposals = proposalsForNonValues(query);
} else { } else {
proposals = foundProposals; proposals = foundProposals;
} }
} }
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults()); final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ", METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query); (System.nanoTime() - start) / 1_000_000.0, query);
return nonEmptyProposals; return nonEmptyProposals;
} }
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) { private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>(); final List<Proposal> proposals = new ArrayList<>();
/* /*
* This method is called when the query could not be parsed. It is likely that * This method is called when the query could not be parsed. It is likely that
* the next word is either a field or an operator. But is is also possible that * the next word is either a field or an operator. But is is also possible that
* the next word is a field-value, because the syntax error might be at another * the next word is a field-value, because the syntax error might be at another
* location in the query (not at the caret position). * location in the query (not at the caret position).
*/ */
final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker()); final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker());
final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER)); final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER));
if (indexTokenWithCaret > 0) { if (indexTokenWithCaret > 0) {
final String previousToken = tokens.get(indexTokenWithCaret - 1); final String previousToken = tokens.get(indexTokenWithCaret - 1);
switch (previousToken) { switch (previousToken) {
case "(": case "(":
case "and": case "and":
case "or": case "or":
case "!": case "!":
proposals.addAll(proposeForAllKeys(query)); proposals.addAll(proposeForAllKeys(query));
break; break;
case ")": case ")":
default: default:
// proposals.addAll(proposal); // proposals.addAll(proposal);
break; break;
} }
} else if (indexTokenWithCaret == 0) { } else if (indexTokenWithCaret == 0) {
proposals.addAll(proposeForAllKeys(query)); proposals.addAll(proposeForAllKeys(query));
} }
return proposals; return proposals;
} }
private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) { private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>(); final List<Proposal> proposals = new ArrayList<>();
final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker()); final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker());
if (wordPrefix != null) { if (wordPrefix != null) {
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange()); final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange());
for (final String field : allFields) { for (final String field : allFields) {
if (!field.startsWith(wordPrefix)) { if (!field.startsWith(wordPrefix)) {
continue; continue;
} }
final String queryWithCaretMarker = query.getQueryWithCaretMarker(); final String queryWithCaretMarker = query.getQueryWithCaretMarker();
final String proposedQuery = queryWithCaretMarker final String proposedQuery = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* "); .replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* ");
final String newQueryWithCaretMarker = queryWithCaretMarker final String newQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER); .replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER);
final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, ""); final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER); final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition); final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal); proposals.add(proposal);
} }
} }
return proposals; return proposals;
} }
private String wordPrefix(final String queryWithCaretMarker) { private String wordPrefix(final String queryWithCaretMarker) {
final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")"); final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")");
final Matcher matcher = pattern.matcher(queryWithCaretMarker); final Matcher matcher = pattern.matcher(queryWithCaretMarker);
if (matcher.find()) { if (matcher.find()) {
final String group = matcher.group(); final String group = matcher.group();
return group.replace(CARET_MARKER, ""); return group.replace(CARET_MARKER, "");
} }
return null; return null;
} }
private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) { private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) {
final List<Proposal> proposals = new ArrayList<>(); final List<Proposal> proposals = new ArrayList<>();
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange); final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange);
for (final String field : allFields) { for (final String field : allFields) {
final String proposedQuery = field + "=*"; final String proposedQuery = field + "=*";
final String newQuery = field + "="; final String newQuery = field + "=";
final int newCaretPosition = newQuery.length(); final int newCaretPosition = newQuery.length();
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition); final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal); proposals.add(proposal);
} }
return proposals; return proposals;
} }
List<Proposal> proposalsForValues(final QueryWithCaretMarker query) { List<Proposal> proposalsForValues(final QueryWithCaretMarker query) {
try { try {
// Add caret marker, so that we know where the caret is. // Add caret marker, so that we know where the caret is.
// This also makes sure that a query like "name=|" ('|' is the caret) can be // This also makes sure that a query like "name=|" ('|' is the caret) can be
// parsed. // parsed.
// Without the caret marker the query would be "name=", which is not a valid // Without the caret marker the query would be "name=", which is not a valid
// expression. // expression.
final String queryWithCaretMarker = query.getQueryWithCaretMarker(); final String queryWithCaretMarker = query.getQueryWithCaretMarker();
// parse the query // parse the query
final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker); final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker);
// normalize it, so that we can use the queryCompletionIndex to search for // normalize it, so that we can use the queryCompletionIndex to search for
// candidate values // candidate values
final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer(); final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer();
final Expression normalizedExpression = optimizer.normalizeExpression(expression); final Expression normalizedExpression = optimizer.normalizeExpression(expression);
// find all candidate values // find all candidate values
final SortedSet<String> candidateValues = normalizedExpression final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex)); .visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(), final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker); candidateValues, queryWithCaretMarker);
// translate the candidate values to proposals // translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues); final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
return proposals; return proposals;
} catch (final SyntaxException e) { } catch (final SyntaxException e) {
LOGGER.debug("Query ({}) is not valid. This is expected to happen " LOGGER.debug("Query ({}) is not valid. This is expected to happen "
+ "unless we are looking for proposals of values.", query, e); + "unless we are looking for proposals of values.", query, e);
return Collections.emptyList(); return Collections.emptyList();
} }
} }
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues, private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) { final String queryWithCaretMarker) {
switch (resultMode) { switch (resultMode) {
case CUT_AT_DOT: case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker); return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES: case FULL_VALUES:
return candidateValues; return candidateValues;
default: default:
throw new IllegalArgumentException("Unexpected value: " + resultMode); throw new IllegalArgumentException("Unexpected value: " + resultMode);
} }
} }
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) { private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper(); final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker); return grouper.group(candidateValues, queryWithCaretMarker);
} }
private List<Proposal> generateProposals(final String queryWithCaretMarker, private List<Proposal> generateProposals(final String queryWithCaretMarker,
final SortedSet<String> candidateValues) { final SortedSet<String> candidateValues) {
final List<Proposal> proposals = new ArrayList<>(); final List<Proposal> proposals = new ArrayList<>();
for (final String proposedTag : candidateValues) { for (final String proposedTag : candidateValues) {
final String proposedQueryWithCaretMarker = queryWithCaretMarker final String proposedQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER); .replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER);
final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, ""); final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER); final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition); final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition);
proposals.add(proposal); proposals.add(proposal);
} }
return proposals; return proposals;
} }
} }

368
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionExpressionOptimizer.java
View File

@@ -45,228 +45,228 @@ import org.slf4j.LoggerFactory;
*/ */
public class QueryCompletionExpressionOptimizer { public class QueryCompletionExpressionOptimizer {
private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class); private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class);
private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor { private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final InExpression expression) { public Expression visit(final InExpression expression) {
if (expression.containsCaret() || expression.getValues().size() == 1) { if (expression.containsCaret() || expression.getValues().size() == 1) {
final String property = expression.getProperty(); final String property = expression.getProperty();
final List<String> values = expression.getValues(); final List<String> values = expression.getValues();
final List<Property> propertyExpressions = new ArrayList<>(); final List<Property> propertyExpressions = new ArrayList<>();
for (final String value : values) { for (final String value : values) {
propertyExpressions.add(new Property(property, new Terminal(value))); propertyExpressions.add(new Property(property, new Terminal(value)));
} }
return Expression.Or.create(propertyExpressions); return Expression.Or.create(propertyExpressions);
} else { } else {
return super.visit(expression); return super.visit(expression);
} }
}; };
} }
private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor { private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final Or expression) { public Expression visit(final Or expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
if (left.containsCaret() && !right.containsCaret()) { if (left.containsCaret() && !right.containsCaret()) {
return left; return left;
} }
if (!left.containsCaret() && right.containsCaret()) { if (!left.containsCaret() && right.containsCaret()) {
return right; return right;
} }
return super.visit(expression); return super.visit(expression);
}; };
} }
private static final class DistributiveNormalization extends IdentityExpressionVisitor { private static final class DistributiveNormalization extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final And expression) { public Expression visit(final And expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
if (left instanceof Or) { if (left instanceof Or) {
// (a or b) and c // (a or b) and c
// becomes // becomes
// a and c or b and c // a and c or b and c
final Expression ac = new And(((Or) left).getLeft(), right); final Expression ac = new And(((Or) left).getLeft(), right);
final Expression bc = new And(((Or) left).getRight(), right); final Expression bc = new And(((Or) left).getRight(), right);
return new Or(ac, bc); return new Or(ac, bc);
} }
if (right instanceof Or) { if (right instanceof Or) {
// a and (b or c) // a and (b or c)
// becomes // becomes
// a and b or a and c // a and b or a and c
final Expression ab = new And(left, ((Or) right).getLeft()); final Expression ab = new And(left, ((Or) right).getLeft());
final Expression ac = new And(left, ((Or) right).getRight()); final Expression ac = new And(left, ((Or) right).getRight());
return new Or(ab, ac); return new Or(ab, ac);
} }
return super.visit(expression); return super.visit(expression);
}; };
} }
private static final class RotateAndExpressions extends IdentityExpressionVisitor { private static final class RotateAndExpressions extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final And expression) { public Expression visit(final And expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
// (| and a) and b => | and (a and b) // (| and a) and b => | and (a and b)
// //
// The expression with the caret is moved up // The expression with the caret is moved up
if (left.containsCaret() && left instanceof And) { if (left.containsCaret() && left instanceof And) {
final Expression leftLeft = ((And) left).getLeft(); final Expression leftLeft = ((And) left).getLeft();
final Expression leftRight = ((And) left).getRight(); final Expression leftRight = ((And) left).getRight();
if (leftLeft.containsCaret()) { if (leftLeft.containsCaret()) {
return new And(leftLeft, new And(leftRight, right)); return new And(leftLeft, new And(leftRight, right));
} else { } else {
return new And(new And(leftLeft, right), leftRight); return new And(new And(leftLeft, right), leftRight);
} }
} else if (right.containsCaret() && right instanceof And) { } else if (right.containsCaret() && right instanceof And) {
final Expression rightLeft = ((And) right).getLeft(); final Expression rightLeft = ((And) right).getLeft();
final Expression rightRight = ((And) right).getRight(); final Expression rightRight = ((And) right).getRight();
if (rightLeft.containsCaret()) { if (rightLeft.containsCaret()) {
return new And(rightLeft, new And(rightRight, left)); return new And(rightLeft, new And(rightRight, left));
} else { } else {
return new And(new And(rightLeft, left), rightRight); return new And(new And(rightLeft, left), rightRight);
} }
} }
return super.visit(expression); return super.visit(expression);
} }
} }
private static final class DoubleNegationExpressions extends IdentityExpressionVisitor { private static final class DoubleNegationExpressions extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final Not expression) { public Expression visit(final Not expression) {
if (expression instanceof Not) { if (expression instanceof Not) {
if (expression.getExpression() instanceof Not) { if (expression.getExpression() instanceof Not) {
return ((Not) expression.getExpression()).getExpression(); return ((Not) expression.getExpression()).getExpression();
} }
} }
return super.visit(expression); return super.visit(expression);
} }
} }
private static final class DeMorgan extends IdentityExpressionVisitor { private static final class DeMorgan extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final Not expression) { public Expression visit(final Not expression) {
if (expression.getExpression() instanceof And) { if (expression.getExpression() instanceof And) {
final And andExpression = (And) expression.getExpression(); final And andExpression = (And) expression.getExpression();
final Expression left = andExpression.getLeft(); final Expression left = andExpression.getLeft();
final Expression right = andExpression.getRight(); final Expression right = andExpression.getRight();
final Expression notLeft = new Not(left); final Expression notLeft = new Not(left);
final Expression notRight = new Not(right); final Expression notRight = new Not(right);
return new Or(notLeft, notRight); return new Or(notLeft, notRight);
} }
return super.visit(expression); return super.visit(expression);
} }
} }
private static final class ToAndCaretExpressions extends IdentityExpressionVisitor { private static final class ToAndCaretExpressions extends IdentityExpressionVisitor {
@Override @Override
public Expression visit(final And expression) { public Expression visit(final And expression) {
final Expression left = expression.getLeft(); final Expression left = expression.getLeft();
final Expression right = expression.getRight(); final Expression right = expression.getRight();
if (left.containsCaret() && left instanceof Property) { if (left.containsCaret() && left instanceof Property) {
return new AndCaretExpression((Property) left, right); return new AndCaretExpression((Property) left, right);
} }
if (right.containsCaret() && right instanceof Property) { if (right.containsCaret() && right instanceof Property) {
return new AndCaretExpression((Property) right, left); return new AndCaretExpression((Property) right, left);
} }
if (left.containsCaret()// if (left.containsCaret()//
&& left instanceof Not// && left instanceof Not//
&& ((Not) left).getExpression() instanceof Property) { && ((Not) left).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right); return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right);
} }
if (right.containsCaret()// if (right.containsCaret()//
&& right instanceof Not// && right instanceof Not//
&& ((Not) right).getExpression() instanceof Property) { && ((Not) right).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left); return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left);
} }
return super.visit(expression); return super.visit(expression);
} }
} }
public Expression normalizeExpression(final Expression expression) { public Expression normalizeExpression(final Expression expression) {
Expression normalizingExpression = expression; Expression normalizingExpression = expression;
Expression previousExpression = normalizingExpression; Expression previousExpression = normalizingExpression;
do { do {
previousExpression = normalizingExpression; previousExpression = normalizingExpression;
// replace all IN-expression, because they are just syntactic sugar for // replace all IN-expression, because they are just syntactic sugar for
// OR-expressions, but only for those that include the caret // OR-expressions, but only for those that include the caret
normalizingExpression = normalizingExpression normalizingExpression = normalizingExpression
.visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor()); .visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor());
// Remove expressions that are OR'ed with the one that contains the caret. // Remove expressions that are OR'ed with the one that contains the caret.
// Everything that is OR'ed with the 'caret'-expression cannot change the // Everything that is OR'ed with the 'caret'-expression cannot change the
// possible values. // possible values.
normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions()); normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions());
// In the end we want to have expressions like "firstname=Jane and lastname=|". // In the end we want to have expressions like "firstname=Jane and lastname=|".
// To reach that goal we use the distributive law to modify expressions like // To reach that goal we use the distributive law to modify expressions like
// "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and // "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and
// lastname=|) or (firstname=John and lastname=|)" // lastname=|) or (firstname=John and lastname=|)"
normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization()); normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization());
// (fn=John and (fn=John and ln=|) // (fn=John and (fn=John and ln=|)
// normalized to // normalized to
// (fn=John and ln=|) and (fn=Jane and ln=|) // (fn=John and ln=|) and (fn=Jane and ln=|)
// or normalized to // or normalized to
// (fn=John and fn=Jane) and ln=| // (fn=John and fn=Jane) and ln=|
normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions()); normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions());
// normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might // normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might
// later be removed // later be removed
// not ( a and b) => (not a) or (not b) // not ( a and b) => (not a) or (not b)
normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan()); normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan());
// remove double negation // remove double negation
// not not a => a // not not a => a
normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions()); normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions());
} while (!normalizingExpression.equals(previousExpression)); } while (!normalizingExpression.equals(previousExpression));
// Replaces all (a and |) expressions with a special expression that represents // Replaces all (a and |) expressions with a special expression that represents
// it. // it.
// This special expression will then be used during evaluation. // This special expression will then be used during evaluation.
return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions()); return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions());
} }
private static Expression visitRepeatedly(final Expression expression, private static Expression visitRepeatedly(final Expression expression,
final ExpressionVisitor<Expression> visitor) { final ExpressionVisitor<Expression> visitor) {
Expression previousExpression; Expression previousExpression;
Expression result = expression; Expression result = expression;
do { do {
previousExpression = result; previousExpression = result;
result = previousExpression.visit(visitor); result = previousExpression.visit(visitor);
if (!previousExpression.equals(result)) { if (!previousExpression.equals(result)) {
LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName()); LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName());
LOGGER.debug(" in: {}", previousExpression); LOGGER.debug(" in: {}", previousExpression);
LOGGER.debug(" out: {}", result); LOGGER.debug(" out: {}", result);
} }
} while (!previousExpression.equals(result)); } while (!previousExpression.equals(result));
return result; return result;
} }
} }

184
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguage.java
View File

@@ -28,125 +28,125 @@ import org.lucares.utils.CollectionUtils;
public class QueryLanguage { public class QueryLanguage {
public Expression parse(final String input) { public Expression parse(final String input) {
// define the input // define the input
final CharStream in = CharStreams.fromString(input); final CharStream in = CharStreams.fromString(input);
// create lexer and parser // create lexer and parser
final PdbLangLexer lexer = new PdbLangLexer(in); final PdbLangLexer lexer = new PdbLangLexer(in);
lexer.addErrorListener(new ErrorListener()); lexer.addErrorListener(new ErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer); final CommonTokenStream tokens = new CommonTokenStream(lexer);
final PdbLangParser parser = new PdbLangParser(tokens); final PdbLangParser parser = new PdbLangParser(tokens);
parser.addErrorListener(new ErrorListener()); parser.addErrorListener(new ErrorListener());
final Stack<Expression> stack = new Stack<>(); final Stack<Expression> stack = new Stack<>();
// define a listener that is called for every terminals and // define a listener that is called for every terminals and
// non-terminals // non-terminals
final ParseTreeListener listener = new PdbLangBaseListener() { final ParseTreeListener listener = new PdbLangBaseListener() {
@Override @Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) { public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
if (ctx.getText().length() > 255) { if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long"); throw new SyntaxException(ctx, "token too long");
} }
stack.push(new Terminal(ctx.getText())); stack.push(new Terminal(ctx.getText()));
} }
@Override @Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
if (ctx.getText().length() > 255) { if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long"); throw new SyntaxException(ctx, "token too long");
} }
stack.push(new Terminal(ctx.getText())); stack.push(new Terminal(ctx.getText()));
} }
@Override @Override
public void exitNotExpression(final NotExpressionContext ctx) { public void exitNotExpression(final NotExpressionContext ctx) {
final Expression expression = stack.pop(); final Expression expression = stack.pop();
final Expression notExpression = new Not(expression); final Expression notExpression = new Not(expression);
stack.push(notExpression); stack.push(notExpression);
} }
@Override @Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) { public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
final Expression right = stack.pop(); final Expression right = stack.pop();
final TemporaryExpression operation = new AndTemporary(); final TemporaryExpression operation = new AndTemporary();
final Expression left = stack.pop(); final Expression left = stack.pop();
stack.push(operation.toExpression(left, right)); stack.push(operation.toExpression(left, right));
} }
@Override @Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) { public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
final Expression right = stack.pop(); final Expression right = stack.pop();
final TemporaryExpression operation = new OrTemporary(); final TemporaryExpression operation = new OrTemporary();
final Expression left = stack.pop(); final Expression left = stack.pop();
stack.push(operation.toExpression(left, right)); stack.push(operation.toExpression(left, right));
} }
@Override @Override
public void exitListOfPropValues(final ListOfPropValuesContext ctx) { public void exitListOfPropValues(final ListOfPropValuesContext ctx) {
final Expression topStackElement = stack.pop(); final Expression topStackElement = stack.pop();
if (topStackElement instanceof ListOfPropertyValues) { if (topStackElement instanceof ListOfPropertyValues) {
// there are at least two property values in the query // there are at least two property values in the query
// e.g. in the expression "bird in (eagle, pigeon)" // e.g. in the expression "bird in (eagle, pigeon)"
final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement; final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement;
final Terminal nextPropertyValue = (Terminal) stack.pop(); final Terminal nextPropertyValue = (Terminal) stack.pop();
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue, final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue,
existingList); existingList);
stack.push(newListOfPropertyValues); stack.push(newListOfPropertyValues);
} else { } else {
// this is the first or the only value in this list of property values // this is the first or the only value in this list of property values
// e.g. in the expression "bird in (eagle)" // e.g. in the expression "bird in (eagle)"
final Terminal propertyValue = (Terminal) topStackElement; final Terminal propertyValue = (Terminal) topStackElement;
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue); final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue);
stack.push(newListOfPropertyValues); stack.push(newListOfPropertyValues);
} }
} }
@Override @Override
public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) { public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) {
final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop(); final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop();
final Terminal propertyName = (Terminal) stack.pop(); final Terminal propertyName = (Terminal) stack.pop();
final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues()); final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues());
stack.push(inExpression); stack.push(inExpression);
} }
}; };
// Specify our entry point // Specify our entry point
final ParseTree parseTree = parser.start(); final ParseTree parseTree = parser.start();
// Walk it and attach our listener // Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker(); final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree); walker.walk(listener, parseTree);
if (stack.size() != 1) { if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack); throw new RuntimeException("stack should have exactly one element " + stack);
} }
return stack.pop(); return stack.pop();
} }
public static List<String> getTokens(final String input) { public static List<String> getTokens(final String input) {
final CharStream in = CharStreams.fromString(input); final CharStream in = CharStreams.fromString(input);
final PdbLangLexer lexer = new PdbLangLexer(in); final PdbLangLexer lexer = new PdbLangLexer(in);
final CommonTokenStream tokens = new CommonTokenStream(lexer); final CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill(); tokens.fill();
final List<Token> tokenList = tokens.getTokens(); final List<Token> tokenList = tokens.getTokens();
return CollectionUtils.map(tokenList, Token::getText); return CollectionUtils.map(tokenList, Token::getText);
} }
} }

20
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguageParser.java
View File

@@ -3,15 +3,15 @@ package org.lucares.pdb.datastore.lang;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
public class QueryLanguageParser { public class QueryLanguageParser {
public static Expression parse(final String query) { public static Expression parse(final String query) {
final Expression result; final Expression result;
if (StringUtils.isEmpty(query)) { if (StringUtils.isEmpty(query)) {
result = Expression.matchAll(); result = Expression.matchAll();
} else { } else {
final QueryLanguage lang = new QueryLanguage(); final QueryLanguage lang = new QueryLanguage();
result = lang.parse(query); result = lang.parse(query);
} }
return result; return result;
} }
} }

90
data-store/src/main/java/org/lucares/pdb/datastore/lang/SyntaxException.java
View File

@@ -4,61 +4,61 @@ import org.antlr.v4.runtime.ParserRuleContext;
public class SyntaxException extends RuntimeException { public class SyntaxException extends RuntimeException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private int lineStart; private int lineStart;
private int startIndex; private int startIndex;
private int lineStop; private int lineStop;
private int stopIndex; private int stopIndex;
public SyntaxException(final ParserRuleContext context, final String message) { public SyntaxException(final ParserRuleContext context, final String message) {
this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(), this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(),
context.getStop().getStopIndex()); context.getStop().getStopIndex());
} }
public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop, public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) { final int stopIndex) {
super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex)); super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex));
this.lineStart = lineStart; this.lineStart = lineStart;
this.startIndex = startIndex; this.startIndex = startIndex;
this.lineStop = lineStop; this.lineStop = lineStop;
this.stopIndex = stopIndex; this.stopIndex = stopIndex;
} }
private static String generateMessage(final int lineStart, final int startIndex, final int lineStop, private static String generateMessage(final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) { final int stopIndex) {
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex); return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
} }
public int getLineStart() { public int getLineStart() {
return lineStart; return lineStart;
} }
public void setLineStart(final int lineStart) { public void setLineStart(final int lineStart) {
this.lineStart = lineStart; this.lineStart = lineStart;
} }
public int getStartIndex() { public int getStartIndex() {
return startIndex; return startIndex;
} }
public void setStartIndex(final int startIndex) { public void setStartIndex(final int startIndex) {
this.startIndex = startIndex; this.startIndex = startIndex;
} }
public int getLineStop() { public int getLineStop() {
return lineStop; return lineStop;
} }
public void setLineStop(final int lineStop) { public void setLineStop(final int lineStop) {
this.lineStop = lineStop; this.lineStop = lineStop;
} }
public int getStopIndex() { public int getStopIndex() {
return stopIndex; return stopIndex;
} }
public void setStopIndex(final int stopIndex) { public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex; this.stopIndex = stopIndex;
} }
} }

460
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -42,293 +42,293 @@ import org.testng.annotations.Test;
@Test @Test
public class DataStoreTest { public class DataStoreTest {
private Path dataDirectory; private Path dataDirectory;
private DataStore dataStore; private DataStore dataStore;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber; private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
dataStore = null; dataStore = null;
tagsToBlockStorageRootBlockNumber = null; tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null; Tags.STRING_COMPRESSOR = null;
} }
public void testQuery() throws Exception { public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory); dataStore = new DataStore(dataDirectory);
final DateTimeRange dateRange = DateTimeRange.relativeHours(1); final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0); final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim"); final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer"); final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer"); final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny"); final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim"); final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>(); tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim)); tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer)); tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer)); tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny)); tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim)); tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim));
assertSearch(dateRange, "bird=eagle", eagleTim); assertSearch(dateRange, "bird=eagle", eagleTim);
assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim); assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name=Tim", eagleTim, labradorTim); assertSearch(dateRange, "name=Tim", eagleTim, labradorTim);
assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim); assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim);
assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny); assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny);
assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny); assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny);
// a͟n͟d binds stronger than o͟r // a͟n͟d binds stronger than o͟r
assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim); assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim);
assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim); assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim);
// parenthesis override priority of a͟n͟d // parenthesis override priority of a͟n͟d
assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim); assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim);
assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim); assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim);
// wildcards // wildcards
assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer); assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny); assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim); assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim); assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim); assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim);
// 'in' queries // 'in' queries
assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer); assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim); assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim);
assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny); assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim); assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim); assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim);
} }
public void testGetByTags() throws IOException { public void testGetByTags() throws IOException {
dataStore = new DataStore(dataDirectory); dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>(); tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer"); final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer"); final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final ParititionId partitionId = new ParititionId("partitionA"); final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer)); tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer)); tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer); final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer"); Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
} }
public void testBlockAlignment() throws IOException { public void testBlockAlignment() throws IOException {
dataStore = new DataStore(dataDirectory); dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim"); final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim); final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0); Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0);
} }
@DataProvider(name = "providerProposals") @DataProvider(name = "providerProposals")
public Iterator<Object[]> providerProposals() { public Iterator<Object[]> providerProposals() {
final List<Object[]> result = new ArrayList<>(); final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") }); result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") });
// returns Tim, because it is the only dog's name starting with 'Ti' // returns Tim, because it is the only dog's name starting with 'Ti'
result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") }); result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") });
// all cats // all cats
result.add(new Object[] { "type=cat and !name=|", "name", result.add(new Object[] { "type=cat and !name=|", "name",
Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") }); Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") });
// finds nothing, because there are not dogs names neither Jenny, nor Ti* // finds nothing, because there are not dogs names neither Jenny, nor Ti*
result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() }); result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() });
result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name", result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name",
Arrays.asList("Jenny", "Tim") }); Arrays.asList("Jenny", "Tim") });
// all but Jennifer // all but Jennifer
result.add(new Object[] { "!(type=bird) and name=|", "name", result.add(new Object[] { "!(type=bird) and name=|", "name",
Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") }); Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") });
result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") }); result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") });
// DeMorgan // DeMorgan
// TODO should only match "Jenny", because Jenny is the only non-bird name // TODO should only match "Jenny", because Jenny is the only non-bird name
// starting with 'Jen' // starting with 'Jen'
result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") }); result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") });
result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name", result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name",
Arrays.asList("Jennifer", "Jenny", "Tim") }); Arrays.asList("Jennifer", "Jenny", "Tim") });
// not existing field // not existing field
result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() }); result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() });
// not existing value // not existing value
result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() }); result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() });
return result.iterator(); return result.iterator();
} }
@Test(dataProvider = "providerProposals") @Test(dataProvider = "providerProposals")
public void testProposals(final String queryWithCaret, final String field, public void testProposals(final String queryWithCaret, final String field,
final List<String> expectedProposedValues) throws Exception { final List<String> expectedProposedValues) throws Exception {
dataStore = new DataStore(dataDirectory); dataStore = new DataStore(dataDirectory);
final ParititionId partitionId = DateIndexExtension.now(); final ParititionId partitionId = DateIndexExtension.now();
final DateTimeRange dateRange = DateTimeRange.relativeHours(1); final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final List<Tags> tags = Arrays.asList( final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"), Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"), Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"), Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"), Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"), Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"), Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"), Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"), Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"), Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John")); Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
tags.forEach(t -> dataStore.createNewFile(partitionId, t)); tags.forEach(t -> dataStore.createNewFile(partitionId, t));
assertProposals(dateRange, queryWithCaret, field, expectedProposedValues); assertProposals(dateRange, queryWithCaret, field, expectedProposedValues);
} }
public void testIdenticalDatesGoIntoSameFile() throws Exception { public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (final DataStore dataStore = new DataStore(dataDirectory)) { try (final DataStore dataStore = new DataStore(dataDirectory)) {
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli(); final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue"); final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
dataStore.write(timestamp, tags, 1); dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2); dataStore.write(timestamp, tags, 2);
Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache"); Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache");
} }
} }
public static void main(final String[] args) throws IOException, InterruptedException { public static void main(final String[] args) throws IOException, InterruptedException {
final Path dir = Files.createTempDirectory("pdb"); final Path dir = Files.createTempDirectory("pdb");
try (final DataStore dataStore = new DataStore(dir)) { try (final DataStore dataStore = new DataStore(dir)) {
final List<Tags> tags = Arrays.asList( final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"), Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"), "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"), "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"), "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"), Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"), Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"), Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"), Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"), Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John")); Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0); final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0); final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
tags.forEach(t -> dataStore.createNewFile(partitionId, t)); tags.forEach(t -> dataStore.createNewFile(partitionId, t));
final JFrame frame = new JFrame(); final JFrame frame = new JFrame();
final JTextField input = new JTextField(); final JTextField input = new JTextField();
final JTextArea output = new JTextArea(); final JTextArea output = new JTextArea();
final JTextArea info = new JTextArea(); final JTextArea info = new JTextArea();
frame.add(input, BorderLayout.NORTH); frame.add(input, BorderLayout.NORTH);
frame.add(output, BorderLayout.CENTER); frame.add(output, BorderLayout.CENTER);
frame.add(info, BorderLayout.SOUTH); frame.add(info, BorderLayout.SOUTH);
input.setText("type=bird and !subtype=eagle and name="); input.setText("type=bird and !subtype=eagle and name=");
input.addKeyListener(new KeyAdapter() { input.addKeyListener(new KeyAdapter() {
@Override @Override
public void keyReleased(final KeyEvent e) { public void keyReleased(final KeyEvent e) {
final String query = input.getText(); final String query = input.getText();
final int caretIndex = input.getCaretPosition(); final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex, final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT); ResultMode.CUT_AT_DOT);
final List<Proposal> proposals = dataStore.propose(q); final List<Proposal> proposals = dataStore.propose(q);
final StringBuilder out = new StringBuilder(); final StringBuilder out = new StringBuilder();
for (final Proposal proposal : proposals) { for (final Proposal proposal : proposals) {
out.append(proposal.getProposedTag()); out.append(proposal.getProposedTag());
out.append(" "); out.append(" ");
out.append(proposal.getProposedQuery()); out.append(proposal.getProposedQuery());
out.append("\n"); out.append("\n");
} }
final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString(); final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString();
out.append("\n"); out.append("\n");
out.append("\n"); out.append("\n");
out.append("input: " + queryWithCaretMarker); out.append("input: " + queryWithCaretMarker);
output.setText(out.toString()); output.setText(out.toString());
} }
}); });
final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS))); final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS)));
final StringBuilder out = new StringBuilder(); final StringBuilder out = new StringBuilder();
out.append("info\n"); out.append("info\n");
for (final Doc doc : docs) { for (final Doc doc : docs) {
out.append(doc.getTags()); out.append(doc.getTags());
out.append("\n"); out.append("\n");
} }
info.setText(out.toString()); info.setText(out.toString());
frame.setSize(800, 600); frame.setSize(800, 600);
frame.setVisible(true); frame.setVisible(true);
TimeUnit.HOURS.sleep(1000); TimeUnit.HOURS.sleep(1000);
} }
} }
private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field, private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field,
final List<String> expectedProposedValues) { final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", ""); final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|"); final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT)); .propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println( System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList())); "proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));
proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery())); proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery()));
final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag); final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag);
Collections.sort(proposedValues); Collections.sort(proposedValues);
Collections.sort(expectedProposedValues); Collections.sort(expectedProposedValues);
Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:"); Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:");
} }
private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) { private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) {
final List<Doc> result = dataStore.search(new Query(query, dateRange)); final List<Doc> result = dataStore.search(new Query(query, dateRange));
Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't."); Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't.");
} }
private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) { private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) {
final Query query = new Query(queryString, dateRange); final Query query = new Query(queryString, dateRange);
final List<Doc> actualDocs = dataStore.search(query); final List<Doc> actualDocs = dataStore.search(query);
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber); final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get); final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual); Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual);
} }
} }

198
data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java
View File

@@ -16,129 +16,129 @@ import org.testng.annotations.Test;
@Test @Test
public class DateIndexExtensionTest { public class DateIndexExtensionTest {
@DataProvider @DataProvider
public Object[][] provider() { public Object[][] provider() {
final List<Object[]> result = new ArrayList<>(); final List<Object[]> result = new ArrayList<>();
{ {
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801"); final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected }); result.add(new Object[] { start, end, expected });
} }
{ {
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802"); final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected }); result.add(new Object[] { start, end, expected });
} }
{ {
// check that adding one month to Jan 31 does not skip the February // check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC); final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803"); final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected }); result.add(new Object[] { start, end, expected });
} }
return result.toArray(new Object[0][]); return result.toArray(new Object[0][]);
} }
@Test(dataProvider = "provider") @Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) { public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
final DateTimeRange dateRange = new DateTimeRange(start, end); final DateTimeRange dateRange = new DateTimeRange(start, end);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange); final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
Assert.assertEquals(actual, expected); Assert.assertEquals(actual, expected);
} }
public void testDateToDateIndexPrefix() { public void testDateToDateIndexPrefix() {
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant() final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli(); .toEpochMilli();
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712"); Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801"); Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801"); Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711"); Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
} }
public void testDateRanges() { public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC) final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2)); .withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC) final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8)); .withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC) final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12)); .withOffsetSameInstant(ZoneOffset.ofHours(12));
final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802); final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802);
final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801); final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801);
final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712); final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712);
final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension
.toPartitionIds(range_201712_201802); .toPartitionIds(range_201712_201802);
Assert.assertEquals(dateIndexPrefixesWithEmptyCache, Assert.assertEquals(dateIndexPrefixesWithEmptyCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802"))); Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")));
final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension
.toPartitionIds(range_201712_201801); .toPartitionIds(range_201712_201801);
Assert.assertEquals(dateIndexPrefixesWithFilledCache, Assert.assertEquals(dateIndexPrefixesWithFilledCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"))); Arrays.asList(new ParititionId("201712"), new ParititionId("201801")));
final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712); final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712);
Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712"))); Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712")));
} }
public void testDateRangeToEpochMilli() { public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3)); final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7)); final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final List<Long> dateIndexEpochMillis = DateIndexExtension final List<Long> dateIndexEpochMillis = DateIndexExtension
.toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802)); .toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802));
Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802)); Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802));
} }
public void testPerformance() { public void testPerformance() {
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final int iterations = 1_000_000; final int iterations = 1_000_000;
final int factor = 1; final int factor = 1;
final int warmup = 20 * factor; final int warmup = 20 * factor;
final int rounds = warmup + 20; final int rounds = warmup + 20;
// fill the cache // fill the cache
DateIndexExtension.DATE_PREFIX_CACHE.clear(); DateIndexExtension.DATE_PREFIX_CACHE.clear();
for (long i = min; i < max; i += 3600 * 24 * 28) { for (long i = min; i < max; i += 3600 * 24 * 28) {
DateIndexExtension.toPartitionId(i); DateIndexExtension.toPartitionId(i);
} }
final List<Double> measurements = new ArrayList<>(); final List<Double> measurements = new ArrayList<>();
for (int r = 0; r < rounds; r++) { for (int r = 0; r < rounds; r++) {
final long start = System.nanoTime(); final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) { for (int i = 0; i < iterations; i++) {
DateIndexExtension.toPartitionId(mid); DateIndexExtension.toPartitionId(mid);
} }
final double duration = (System.nanoTime() - start) / 1_000_000.0; final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms"); System.out.println("duration: " + duration + "ms");
measurements.add(duration); measurements.add(duration);
} }
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d) final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics(); .summaryStatistics();
System.out.println(stats); System.out.println(stats);
} }
} }

465
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -22,275 +22,276 @@ import org.testng.annotations.Test;
@Test @Test
public class ProposerTest { public class ProposerTest {
private Path dataDirectory; private Path dataDirectory;
private DataStore dataStore; private DataStore dataStore;
private DateTimeRange dateRange; private DateTimeRange dateRange;
@BeforeClass @BeforeClass
public void beforeClass() throws Exception { public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
initDatabase(); initDatabase();
} }
@AfterClass @AfterClass
public void afterClass() throws IOException { public void afterClass() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
dataStore.close(); dataStore.close();
dataStore = null; dataStore = null;
Tags.STRING_COMPRESSOR = null; Tags.STRING_COMPRESSOR = null;
} }
private void initDatabase() throws Exception { private void initDatabase() throws Exception {
dataStore = new DataStore(dataDirectory); dataStore = new DataStore(dataDirectory);
dateRange = DateTimeRange.now(); dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0); final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim"); final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy"); final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer"); final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer"); final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny"); final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim"); final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web"); final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source", final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service"); "service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web"); final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service"); final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
dataStore.createNewFile(now, eagleTim); dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy); dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer); dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer); dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny); dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim); dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, methodA); dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB); dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC); dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD); dataStore.createNewFile(now, methodD);
} }
public void testEmptyQuery() throws Exception { public void testEmptyQuery() throws Exception {
assertProposals("|", ResultMode.FULL_VALUES, // assertProposals("|", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), // new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), // new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), // new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), // new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)// new Proposal("source", "source=*", true, "source=", 7)//
); );
assertProposals(" |", ResultMode.FULL_VALUES, // assertProposals(" |", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), // new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), // new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), // new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), // new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)// new Proposal("source", "source=*", true, "source=", 7)//
); );
} }
public void testPrefixOfKey() throws Exception { public void testPrefixOfKey() throws Exception {
assertProposals("bi|", ResultMode.FULL_VALUES, // assertProposals("bi|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) // new Proposal("bird", "bird=* ", true, "bird=", 5) //
); );
assertProposals("bird|", ResultMode.FULL_VALUES, // assertProposals("bird|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) // new Proposal("bird", "bird=* ", true, "bird=", 5) //
); );
assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, // assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) // new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
); );
assertProposals("|bird", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
public void testPrefixOfValue() throws Exception { assertProposals("|bird", ResultMode.FULL_VALUES, //
assertProposals("name =Tim|", ResultMode.FULL_VALUES, // new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9), new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13)); new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
assertProposals("name =Je|", ResultMode.FULL_VALUES, // public void testPrefixOfValue() throws Exception {
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), // assertProposals("name =Tim|", ResultMode.FULL_VALUES, //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) // new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
); new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), // assertProposals("name =Je|", ResultMode.FULL_VALUES, //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) // new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
); new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
// TODO this case is currently handled completely wrong - it is handled similar to an empty query assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
// TODO this case is currently handled completely wrong - it is handled similar
// to an empty query
// assertProposals("|bird=eagle and name=Tim", ResultMode.FULL_VALUES, // // assertProposals("|bird=eagle and name=Tim", ResultMode.FULL_VALUES, //
// new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), // // new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
// new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) // // new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
// ); // );
/*
*/
}
@Test(enabled = true) /*
public void testInExpressions() throws Exception { */
assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, // }
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, // @Test(enabled = true)
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), // public void testInExpressions() throws Exception {
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22)); assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, // assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11), new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15)); new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
/* assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, //
*/ new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
} new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
public void testProposalOnEmptyValuePrefix() throws Exception { /*
assertProposals("name=|", ResultMode.FULL_VALUES, // */
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), // }
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
assertProposals("method=|", ResultMode.CUT_AT_DOT, // public void testProposalOnEmptyValuePrefix() throws Exception {
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), // assertProposals("name=|", ResultMode.FULL_VALUES, //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), // new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), // new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) // new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
); new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
assertProposals("method=|", ResultMode.FULL_VALUES, // );
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
public void testProposalOnValueSmartExpression() throws Exception { assertProposals("method=|", ResultMode.CUT_AT_DOT, //
assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, // new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
"method=FooController.doImportantStuff", 37), // new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
"method=FooService.doImportantStuff", 34), // );
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true, assertProposals("method=|", ResultMode.FULL_VALUES, //
"method=FooBarService.doOtherStuff", 33) // new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
); "method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, // public void testProposalOnValueSmartExpression() throws Exception {
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, //
"method=FooController.doImportantStuff", 37), // new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, "method=FooController.doImportantStuff", 37), //
"method=FooService.doImportantStuff", 34), // new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true, "method=FooService.doImportantStuff", 34), //
"method=FooBarService.doOtherStuff", 33) // new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
); "method=FooBarService.doOtherStuff", 33) //
);
// returns nothing, because GloblikePattern.globlikeToRegex() returns the assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, //
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
// Maybe I will change that some day and allow upper case characters before "method=FooController.doImportantStuff", 37), //
// "Stuff". new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT); "method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, // // returns nothing, because GloblikePattern.globlikeToRegex() returns the
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, // following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
"method=FooController.doImportantStuff", 37), // // Maybe I will change that some day and allow upper case characters before
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, // "Stuff".
"method=FooService.doImportantStuff", 34) // assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT);
);
}
public void testProposalOnEmptyKeyPrefix() throws Exception { assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
assertProposals("name=* and |", ResultMode.FULL_VALUES, // new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
proposal("name", "name=* and name=* ", "name=* and name=|"), // "method=FooController.doImportantStuff", 37), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), // new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), // "method=FooService.doImportantStuff", 34) //
// TODO it is wrong to return those two, because there are no values with name );
// and type|address, but I'll leave this for now, because this is a different }
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
public void testProposalWithWildcards() throws Exception { public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and |", ResultMode.FULL_VALUES, //
proposal("name", "name=* and name=* ", "name=* and name=|"), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), //
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
assertProposals("name=*im|", ResultMode.FULL_VALUES, // public void testProposalWithWildcards() throws Exception {
proposal("Tim", "name=Tim", "name=Tim|"), //
proposal("Timothy", "name=Timothy", "name=Timothy|")//
);
assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, // assertProposals("name=*im|", ResultMode.FULL_VALUES, //
proposal("FooService.doImportantStuff", proposal("Tim", "name=Tim", "name=Tim|"), //
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff", proposal("Timothy", "name=Timothy", "name=Timothy|")//
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), // );
proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
public void testProposalWithAndExpression() throws Exception { assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, //
assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, // proposal("FooService.doImportantStuff",
proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), // "(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff",
proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")// "(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), //
); proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, // public void testProposalWithAndExpression() throws Exception {
proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), // assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, //
proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")// proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), //
); proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")//
} );
public void testProposalWithAndNotExpression() throws Exception { assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, //
assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, // proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), //
proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle", proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")//
"name=Tim and ! dog=labrador and bird=eagle|") // );
); }
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) { public void testProposalWithAndNotExpression() throws Exception {
final String newQueryWithoutCaretMarker = newQuery.replace("|", ""); assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, //
final int newCaretPosition = newQuery.indexOf('|'); proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle",
return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition); "name=Tim and ! dog=labrador and bird=eagle|") //
} );
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected) private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) {
throws InterruptedException { final String newQueryWithoutCaretMarker = newQuery.replace("|", "");
final int caretIndex = query.indexOf("|"); final int newCaretPosition = newQuery.indexOf('|');
final String q = query.replace("|", ""); return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition);
assertProposals(q, resultMode, caretIndex, expected); }
}
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex, private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected)
final Proposal... expected) throws InterruptedException { throws InterruptedException {
final int caretIndex = query.indexOf("|");
final String q = query.replace("|", "");
assertProposals(q, resultMode, caretIndex, expected);
}
final List<Proposal> actual = dataStore private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode)); final Proposal... expected) throws InterruptedException {
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---"); final List<Proposal> actual = dataStore
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString))); .propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString))); final List<Proposal> expectedList = Arrays.asList(expected);
Assert.assertEquals(actual, expectedList); Collections.sort(expectedList);
}
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString)));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString)));
Assert.assertEquals(actual, expectedList);
}
} }

78
data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java
View File

@@ -21,53 +21,53 @@ import org.testng.annotations.Test;
@Test @Test
public class QueryCompletionIndexTest { public class QueryCompletionIndexTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void test() throws Exception { public void test() throws Exception {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs()); Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
final List<Tags> tags = Arrays.asList(// final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
); );
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1); final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0); final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) { try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) {
for (final Tags t : tags) { for (final Tags t : tags) {
index.addTags(partitionId, t); index.addTags(partitionId, t);
} }
// all firstnames where lastname=Doe are returned sorted alphabetically. // all firstnames where lastname=Doe are returned sorted alphabetically.
// tags A and B match // tags A and B match
final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"), final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"),
"firstname"); "firstname");
Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John")); Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John"));
// no duplicates are returned: // no duplicates are returned:
// tags A and C match firstname=John, but both have country=Atlantis // tags A and C match firstname=John, but both have country=Atlantis
final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"), final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"),
"country"); "country");
Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis")); Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis"));
// findAllValuesForField sorts alphabetically // findAllValuesForField sorts alphabetically
final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname"); final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname");
Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames); Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames);
final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country"); final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country");
Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado")); Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado"));
} }
} }
} }

88
data-store/src/test/java/org/lucares/pdb/datastore/lang/CandidateGrouperTest.java
View File

@@ -12,54 +12,54 @@ import org.testng.annotations.Test;
@Test @Test
public class CandidateGrouperTest { public class CandidateGrouperTest {
@DataProvider @DataProvider
public Object[][] providerGroup() { public Object[][] providerGroup() {
final List<Object[]> result = new ArrayList<>(); final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = |", // "name = |", //
Set.of("aa.") }); Set.of("aa.") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = a|", // "name = a|", //
Set.of("aa.") }); Set.of("aa.") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa|", // "name = aa|", //
Set.of("aa.") }); Set.of("aa.") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.|", // "name = aa.|", //
Set.of("aa.xx.", "aa.yy.") }); Set.of("aa.xx.", "aa.yy.") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.x|", // "name = aa.x|", //
Set.of("aa.xx.") }); Set.of("aa.xx.") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.xx.|", // "name = aa.xx.|", //
Set.of("aa.xx.AA.", "aa.xx.BB") }); Set.of("aa.xx.AA.", "aa.xx.BB") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), // Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), //
"name = aa.xx.AA.|", // "name = aa.xx.AA.|", //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") }); Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") });
result.add(new Object[] { // result.add(new Object[] { //
Set.of("XX.YY.ZZ", "XX.YY"), // Set.of("XX.YY.ZZ", "XX.YY"), //
"name = XX.Y|", // "name = XX.Y|", //
Set.of("XX.YY.", "XX.YY") }); Set.of("XX.YY.", "XX.YY") });
return result.toArray(new Object[0][]); return result.toArray(new Object[0][]);
} }
@Test(dataProvider = "providerGroup") @Test(dataProvider = "providerGroup")
public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) { public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) {
final CandidateGrouper grouper = new CandidateGrouper(); final CandidateGrouper grouper = new CandidateGrouper();
final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER); final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER);
final SortedSet<String> actual = grouper.group(values, query); final SortedSet<String> actual = grouper.group(values, query);
Assert.assertEquals(actual, expected); Assert.assertEquals(actual, expected);
} }
} }

76
file-utils/src/main/java/org/lucares/utils/file/FileUtils.java
View File

@@ -15,55 +15,55 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class FileUtils { public class FileUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
private static final class RecursiveDeleter extends SimpleFileVisitor<Path> { private static final class RecursiveDeleter extends SimpleFileVisitor<Path> {
@Override @Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
Files.delete(file); Files.delete(file);
LOGGER.trace("deleted: {}", file); LOGGER.trace("deleted: {}", file);
return FileVisitResult.CONTINUE; return FileVisitResult.CONTINUE;
} }
@Override @Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
Files.delete(dir); Files.delete(dir);
LOGGER.trace("deleted: {}", dir); LOGGER.trace("deleted: {}", dir);
return FileVisitResult.CONTINUE; return FileVisitResult.CONTINUE;
} }
} }
public static void delete(final Path path) { public static void delete(final Path path) {
final int maxAttempts = 10; final int maxAttempts = 10;
int attempt = 1; int attempt = 1;
while (attempt <= maxAttempts) { while (attempt <= maxAttempts) {
try { try {
LOGGER.debug("deleting '{}' attempt {} of {}", path.toFile().getAbsolutePath(), attempt, maxAttempts); LOGGER.debug("deleting '{}' attempt {} of {}", path.toFile().getAbsolutePath(), attempt, maxAttempts);
Files.walkFileTree(path, new RecursiveDeleter()); Files.walkFileTree(path, new RecursiveDeleter());
break; break;
} catch (final IOException e) { } catch (final IOException e) {
final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt
+ " of " + maxAttempts; + " of " + maxAttempts;
LOGGER.warn(msg, e); LOGGER.warn(msg, e);
} }
attempt++; attempt++;
} }
} }
public static List<Path> listRecursively(final Path start) throws IOException { public static List<Path> listRecursively(final Path start) throws IOException {
final int maxDepth = Integer.MAX_VALUE; final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path); final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) { try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
return files.collect(Collectors.toList()); return files.collect(Collectors.toList());
} }
} }
} }

156
pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java
View File

@@ -8,106 +8,106 @@ import java.time.temporal.TemporalUnit;
public class DateTimeRange { public class DateTimeRange {
private static final DateTimeRange MAX = new DateTimeRange( private static final DateTimeRange MAX = new DateTimeRange(
OffsetDateTime.of(1900, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC), OffsetDateTime.of(1900, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC),
OffsetDateTime.of(2100, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)); OffsetDateTime.of(2100, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC));
private final OffsetDateTime start; private final OffsetDateTime start;
private final OffsetDateTime end; private final OffsetDateTime end;
public DateTimeRange(final OffsetDateTime start, final OffsetDateTime end) { public DateTimeRange(final OffsetDateTime start, final OffsetDateTime end) {
this.start = start; this.start = start;
this.end = end; this.end = end;
} }
public static DateTimeRange max() { public static DateTimeRange max() {
return MAX; return MAX;
} }
public static DateTimeRange now() { public static DateTimeRange now() {
return relativeMillis(0); return relativeMillis(0);
} }
public static DateTimeRange relative(final long amount, final TemporalUnit unit) { public static DateTimeRange relative(final long amount, final TemporalUnit unit) {
final OffsetDateTime now = OffsetDateTime.now(); final OffsetDateTime now = OffsetDateTime.now();
return new DateTimeRange(now.minus(amount, unit), now); return new DateTimeRange(now.minus(amount, unit), now);
} }
public static DateTimeRange relativeMillis(final long amount) { public static DateTimeRange relativeMillis(final long amount) {
return relative(amount, ChronoUnit.MILLIS); return relative(amount, ChronoUnit.MILLIS);
} }
public static DateTimeRange relativeSeconds(final long amount) { public static DateTimeRange relativeSeconds(final long amount) {
return relative(amount, ChronoUnit.SECONDS); return relative(amount, ChronoUnit.SECONDS);
} }
public static DateTimeRange relativeMinutes(final long amount) { public static DateTimeRange relativeMinutes(final long amount) {
return relative(amount, ChronoUnit.MINUTES); return relative(amount, ChronoUnit.MINUTES);
} }
public static DateTimeRange relativeHours(final long amount) { public static DateTimeRange relativeHours(final long amount) {
return relative(amount, ChronoUnit.HOURS); return relative(amount, ChronoUnit.HOURS);
} }
public static DateTimeRange relativeDays(final long amount) { public static DateTimeRange relativeDays(final long amount) {
return relative(amount, ChronoUnit.DAYS); return relative(amount, ChronoUnit.DAYS);
} }
public static DateTimeRange relativeMonths(final long amount) { public static DateTimeRange relativeMonths(final long amount) {
return relative(amount, ChronoUnit.MONTHS); return relative(amount, ChronoUnit.MONTHS);
} }
public static DateTimeRange relativeYears(final long amount) { public static DateTimeRange relativeYears(final long amount) {
return relative(amount, ChronoUnit.YEARS); return relative(amount, ChronoUnit.YEARS);
} }
public OffsetDateTime getStart() { public OffsetDateTime getStart() {
return start; return start;
} }
public long getStartEpochMilli() { public long getStartEpochMilli() {
return start.toInstant().toEpochMilli(); return start.toInstant().toEpochMilli();
} }
public OffsetDateTime getEnd() { public OffsetDateTime getEnd() {
return end; return end;
} }
public long getEndEpochMilli() { public long getEndEpochMilli() {
return end.toInstant().toEpochMilli(); return end.toInstant().toEpochMilli();
} }
@Override @Override
public String toString() { public String toString() {
return start + "-" + end; return start + "-" + end;
} }
public static DateTimeRange ofDay(final OffsetDateTime day) { public static DateTimeRange ofDay(final OffsetDateTime day) {
final OffsetDateTime from = day.truncatedTo(ChronoUnit.DAYS); final OffsetDateTime from = day.truncatedTo(ChronoUnit.DAYS);
final OffsetDateTime to = from.plusDays(1).minusNanos(1); final OffsetDateTime to = from.plusDays(1).minusNanos(1);
return new DateTimeRange(from, to); return new DateTimeRange(from, to);
} }
public Duration duration() { public Duration duration() {
return Duration.between(start, end); return Duration.between(start, end);
} }
public boolean inRange(final long epochMilli) { public boolean inRange(final long epochMilli) {
final long fromEpochMilli = start.toInstant().toEpochMilli(); final long fromEpochMilli = start.toInstant().toEpochMilli();
final long toEpochMilli = end.toInstant().toEpochMilli(); final long toEpochMilli = end.toInstant().toEpochMilli();
return fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli; return fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli;
} }
public boolean inRange(final OffsetDateTime date) { public boolean inRange(final OffsetDateTime date) {
return start.compareTo(date) <= 0 && end.compareTo(date) >= 0; return start.compareTo(date) <= 0 && end.compareTo(date) >= 0;
} }
public boolean intersect(final DateTimeRange timeRange) { public boolean intersect(final DateTimeRange timeRange) {
return inRange(timeRange.start) // return inRange(timeRange.start) //
|| inRange(timeRange.end) // || inRange(timeRange.end) //
|| timeRange.inRange(start)// || timeRange.inRange(start)//
|| timeRange.inRange(end); || timeRange.inRange(end);
} }
} }

50
pdb-api/src/main/java/org/lucares/pdb/api/Entries.java
View File

@@ -7,36 +7,36 @@ import java.util.Iterator;
import java.util.List; import java.util.List;
public class Entries implements Iterable<Entry> { public class Entries implements Iterable<Entry> {
/** /**
* A special {@link Entries} instance that can be used as poison object for * A special {@link Entries} instance that can be used as poison object for
* {@link BlockingQueueIterator}. * {@link BlockingQueueIterator}.
*/ */
public static final Entries POISON = new Entries(0); public static final Entries POISON = new Entries(0);
private final List<Entry> entries; private final List<Entry> entries;
public Entries(final int initialSize) { public Entries(final int initialSize) {
entries = new ArrayList<>(initialSize); entries = new ArrayList<>(initialSize);
} }
public Entries(final Entry... entries) { public Entries(final Entry... entries) {
this.entries = new ArrayList<>(Arrays.asList(entries)); this.entries = new ArrayList<>(Arrays.asList(entries));
} }
public Entries(final Collection<Entry> entries) { public Entries(final Collection<Entry> entries) {
this.entries = new ArrayList<>(entries); this.entries = new ArrayList<>(entries);
} }
public void add(final Entry entry) { public void add(final Entry entry) {
entries.add(entry); entries.add(entry);
} }
@Override @Override
public Iterator<Entry> iterator() { public Iterator<Entry> iterator() {
return entries.iterator(); return entries.iterator();
} }
public int size() { public int size() {
return entries.size(); return entries.size();
} }
} }

102
pdb-api/src/main/java/org/lucares/pdb/api/Entry.java
View File

@@ -7,65 +7,65 @@ import java.time.format.DateTimeFormatter;
public class Entry { public class Entry {
private final long value; private final long value;
private final Tags tags; private final Tags tags;
private final long epochMilli; private final long epochMilli;
public Entry(final long epochMilli, final long value, final Tags tags) { public Entry(final long epochMilli, final long value, final Tags tags) {
this.epochMilli = epochMilli; this.epochMilli = epochMilli;
this.tags = tags; this.tags = tags;
this.value = value; this.value = value;
} }
public long getValue() { public long getValue() {
return value; return value;
} }
public long getEpochMilli() { public long getEpochMilli() {
return epochMilli; return epochMilli;
} }
public Tags getTags() { public Tags getTags() {
return tags; return tags;
} }
@Override @Override
public String toString() { public String toString() {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags.asString() + ")"; return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags.asString() + ")";
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32)); result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode()); result = prime * result + ((tags == null) ? 0 : tags.hashCode());
result = prime * result + (int) (value ^ (value >>> 32)); result = prime * result + (int) (value ^ (value >>> 32));
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final Entry other = (Entry) obj; final Entry other = (Entry) obj;
if (epochMilli != other.epochMilli) if (epochMilli != other.epochMilli)
return false; return false;
if (tags == null) { if (tags == null) {
if (other.tags != null) if (other.tags != null)
return false; return false;
} else if (!tags.equals(other.tags)) } else if (!tags.equals(other.tags))
return false; return false;
if (value != other.value) if (value != other.value)
return false; return false;
return true; return true;
} }
} }

40
pdb-api/src/main/java/org/lucares/pdb/api/GroupResult.java
View File

@@ -6,31 +6,31 @@ import org.lucares.collections.LongList;
public class GroupResult { public class GroupResult {
private final Tags groupedBy; private final Tags groupedBy;
private final Stream<LongList> timeValueStream; private final Stream<LongList> timeValueStream;
public GroupResult(final Stream<LongList> entries, final Tags groupedBy) { public GroupResult(final Stream<LongList> entries, final Tags groupedBy) {
this.timeValueStream = entries; this.timeValueStream = entries;
this.groupedBy = groupedBy; this.groupedBy = groupedBy;
} }
public Tags getGroupedBy() { public Tags getGroupedBy() {
return groupedBy; return groupedBy;
} }
/** /**
* @return {@link Stream} * @return {@link Stream}
*/ */
public Stream<LongList> asStream() { public Stream<LongList> asStream() {
return timeValueStream; return timeValueStream;
} }
public LongList flatMap() { public LongList flatMap() {
final LongList result = new LongList(); final LongList result = new LongList();
timeValueStream.forEachOrdered(result::addAll); timeValueStream.forEachOrdered(result::addAll);
return result; return result;
} }
} }

102
pdb-api/src/main/java/org/lucares/pdb/api/Query.java
View File

@@ -4,74 +4,74 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
public class Query { public class Query {
private final String query; private final String query;
private final DateTimeRange dateRange; private final DateTimeRange dateRange;
public Query(final String query, final DateTimeRange dateRange) { public Query(final String query, final DateTimeRange dateRange) {
super(); super();
this.query = query; this.query = query;
this.dateRange = dateRange; this.dateRange = dateRange;
} }
public Query relativeMillis(final String query, final long amount) { public Query relativeMillis(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMillis(amount)); return new Query(query, DateTimeRange.relativeMillis(amount));
} }
public Query relativeSeconds(final String query, final long amount) { public Query relativeSeconds(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeSeconds(amount)); return new Query(query, DateTimeRange.relativeSeconds(amount));
} }
public Query relativeMinutes(final String query, final long amount) { public Query relativeMinutes(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMinutes(amount)); return new Query(query, DateTimeRange.relativeMinutes(amount));
} }
public Query relativeHours(final String query, final long amount) { public Query relativeHours(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeHours(amount)); return new Query(query, DateTimeRange.relativeHours(amount));
} }
public Query relativeDays(final String query, final long amount) { public Query relativeDays(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeDays(amount)); return new Query(query, DateTimeRange.relativeDays(amount));
} }
public Query relativeMonths(final String query, final long amount) { public Query relativeMonths(final String query, final long amount) {
return new Query(query, DateTimeRange.relativeMonths(amount)); return new Query(query, DateTimeRange.relativeMonths(amount));
} }
public static Query createQuery(final String query, final DateTimeRange dateRange) { public static Query createQuery(final String query, final DateTimeRange dateRange) {
return new Query(query, dateRange); return new Query(query, dateRange);
} }
public static Query createQuery(final Tags tags, final DateTimeRange dateRange) { public static Query createQuery(final Tags tags, final DateTimeRange dateRange) {
final List<String> terms = new ArrayList<>(); final List<String> terms = new ArrayList<>();
for (final String key : tags.getKeys()) { for (final String key : tags.getKeys()) {
final String value = tags.getValue(key); final String value = tags.getValue(key);
final StringBuilder term = new StringBuilder(); final StringBuilder term = new StringBuilder();
term.append(key); term.append(key);
term.append("="); term.append("=");
term.append(value); term.append(value);
term.append(" "); term.append(" ");
terms.add(term.toString()); terms.add(term.toString());
} }
return new Query(String.join(" and ", terms), dateRange); return new Query(String.join(" and ", terms), dateRange);
} }
public String getQuery() { public String getQuery() {
return query; return query;
} }
public DateTimeRange getDateRange() { public DateTimeRange getDateRange() {
return dateRange; return dateRange;
} }
@Override @Override
public String toString() { public String toString() {
return "'" + query + "' [" + dateRange + "]"; return "'" + query + "' [" + dateRange + "]";
} }
} }

2
pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.api; package org.lucares.pdb.api;
public interface QueryConstants { public interface QueryConstants {
String CARET_MARKER = "\ue001"; // character in the private use area String CARET_MARKER = "\ue001"; // character in the private use area
} }

38
pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java
View File

@@ -2,28 +2,28 @@ package org.lucares.pdb.api;
public class QueryWithCaretMarker extends Query implements QueryConstants { public class QueryWithCaretMarker extends Query implements QueryConstants {
public enum ResultMode { public enum ResultMode {
CUT_AT_DOT, FULL_VALUES CUT_AT_DOT, FULL_VALUES
} }
private final int caretIndex; private final int caretIndex;
private final ResultMode resultMode; private final ResultMode resultMode;
public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex, public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex,
final ResultMode resultMode) { final ResultMode resultMode) {
super(query, dateRange); super(query, dateRange);
this.caretIndex = caretIndex; this.caretIndex = caretIndex;
this.resultMode = resultMode; this.resultMode = resultMode;
} }
public String getQueryWithCaretMarker() { public String getQueryWithCaretMarker() {
final StringBuilder queryBuilder = new StringBuilder(getQuery()); final StringBuilder queryBuilder = new StringBuilder(getQuery());
final StringBuilder queryWithCaretMarker = queryBuilder.insert(caretIndex, CARET_MARKER); final StringBuilder queryWithCaretMarker = queryBuilder.insert(caretIndex, CARET_MARKER);
return queryWithCaretMarker.toString(); return queryWithCaretMarker.toString();
} }
public ResultMode getResultMode() { public ResultMode getResultMode() {
return resultMode; return resultMode;
} }
} }

32
pdb-api/src/main/java/org/lucares/pdb/api/Result.java
View File

@@ -7,24 +7,24 @@ import java.util.List;
public class Result { public class Result {
private final List<GroupResult> groupResults; private final List<GroupResult> groupResults;
public Result(final GroupResult... groupResults) { public Result(final GroupResult... groupResults) {
this(Arrays.asList(groupResults)); this(Arrays.asList(groupResults));
} }
public Result(final Collection<GroupResult> groupResults) { public Result(final Collection<GroupResult> groupResults) {
this.groupResults = new ArrayList<>(groupResults); this.groupResults = new ArrayList<>(groupResults);
} }
public GroupResult singleGroup() { public GroupResult singleGroup() {
if (groupResults.size() != 1) { if (groupResults.size() != 1) {
throw new IllegalStateException("the result does not contain exactly one group"); throw new IllegalStateException("the result does not contain exactly one group");
} }
return groupResults.get(0); return groupResults.get(0);
} }
public List<GroupResult> getGroups() { public List<GroupResult> getGroups() {
return new ArrayList<>(groupResults); return new ArrayList<>(groupResults);
} }
} }

8
pdb-api/src/main/java/org/lucares/pdb/api/RuntimeIOException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.api;
public class RuntimeIOException extends RuntimeException { public class RuntimeIOException extends RuntimeException {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public RuntimeIOException(final Throwable cause) { public RuntimeIOException(final Throwable cause) {
super(cause); super(cause);
} }
} }

42
pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java
View File

@@ -7,34 +7,34 @@ import java.nio.file.Path;
*/ */
public class StringCompressor { public class StringCompressor {
private final UniqueStringIntegerPairs usip; private final UniqueStringIntegerPairs usip;
public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException { public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException {
this.usip = usip; this.usip = usip;
} }
public static StringCompressor create(final Path path) { public static StringCompressor create(final Path path) {
final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path); final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path);
return new StringCompressor(mapsi); return new StringCompressor(mapsi);
} }
public int put(final String string) { public int put(final String string) {
return usip.computeIfAbsent(string, s -> usip.getHighestInteger() + 1); return usip.computeIfAbsent(string, s -> usip.getHighestInteger() + 1);
} }
public int put(final byte[] bytes, final int start, final int endExclusive) { public int put(final byte[] bytes, final int start, final int endExclusive) {
return usip.computeIfAbsent(bytes, start, endExclusive); return usip.computeIfAbsent(bytes, start, endExclusive);
} }
public String get(final int integer) { public String get(final int integer) {
return usip.getKey(integer); return usip.getKey(integer);
} }
public int getIfPresent(final String string) { public int getIfPresent(final String string) {
final Integer integer = usip.get(string); final Integer integer = usip.get(string);
return integer != null ? integer : -1; return integer != null ? integer : -1;
} }
} }

142
pdb-api/src/main/java/org/lucares/pdb/api/Tag.java
View File

@@ -6,89 +6,89 @@ package org.lucares.pdb.api;
* 'Sam' is the value. * 'Sam' is the value.
*/ */
public class Tag implements Comparable<Tag> { public class Tag implements Comparable<Tag> {
private final int field; private final int field;
private final int value; private final int value;
/** /**
* Create a new tag with field and value specified as int. See * Create a new tag with field and value specified as int. See
* {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints. * {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints.
* *
* @param field the field as int * @param field the field as int
* @param value the value as int * @param value the value as int
*/ */
public Tag(final int field, final int value) { public Tag(final int field, final int value) {
this.field = field; this.field = field;
this.value = value; this.value = value;
} }
/** /**
* Create a new {@link Tag} for the given field and value. * Create a new {@link Tag} for the given field and value.
* *
* @param field the field * @param field the field
* @param value the value * @param value the value
*/ */
public Tag(final String field, final String value) { public Tag(final String field, final String value) {
this.field = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1; this.field = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1;
this.value = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1; this.value = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1;
} }
@Override @Override
public int compareTo(final Tag o) { public int compareTo(final Tag o) {
if (field != o.field) { if (field != o.field) {
return field - o.field; return field - o.field;
} else if (value != o.value) { } else if (value != o.value) {
return value - o.value; return value - o.value;
} }
return 0; return 0;
} }
public int getKey() { public int getKey() {
return field; return field;
} }
public String getKeyAsString() { public String getKeyAsString() {
return Tags.STRING_COMPRESSOR.get(field); return Tags.STRING_COMPRESSOR.get(field);
} }
public int getValue() { public int getValue() {
return value; return value;
} }
public String getValueAsString() { public String getValueAsString() {
return Tags.STRING_COMPRESSOR.get(value); return Tags.STRING_COMPRESSOR.get(value);
} }
@Override @Override
public String toString() { public String toString() {
return Tags.STRING_COMPRESSOR.get(field) + "=" + Tags.STRING_COMPRESSOR.get(value); return Tags.STRING_COMPRESSOR.get(field) + "=" + Tags.STRING_COMPRESSOR.get(value);
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + field; result = prime * result + field;
result = prime * result + value; result = prime * result + value;
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final Tag other = (Tag) obj; final Tag other = (Tag) obj;
if (field != other.field) if (field != other.field)
return false; return false;
if (value != other.value) if (value != other.value)
return false; return false;
return true; return true;
} }
} }

2
pdb-api/src/main/java/org/lucares/pdb/api/TagByKeyAndValueComparator.java
View File

@@ -4,5 +4,5 @@ import java.util.Comparator;
public class TagByKeyAndValueComparator { public class TagByKeyAndValueComparator {
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey).thenComparing(Tag::getValue); public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey).thenComparing(Tag::getValue);
} }

2
pdb-api/src/main/java/org/lucares/pdb/api/TagByKeyComparator.java
View File

@@ -4,5 +4,5 @@ import java.util.Comparator;
public class TagByKeyComparator { public class TagByKeyComparator {
public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey); public static final Comparator<Tag> INSTANCE = Comparator.comparing(Tag::getKey);
} }

408
pdb-api/src/main/java/org/lucares/pdb/api/Tags.java
View File

@@ -14,264 +14,264 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
public class Tags implements Comparable<Tags> { public class Tags implements Comparable<Tags> {
public static StringCompressor STRING_COMPRESSOR = null; public static StringCompressor STRING_COMPRESSOR = null;
private static final byte[] EMPTY_BYTES = new byte[0]; private static final byte[] EMPTY_BYTES = new byte[0];
public static final Tags EMPTY = new Tags(); public static final Tags EMPTY = new Tags();
private final List<Tag> tags; private final List<Tag> tags;
public Tags() { public Tags() {
tags = new ArrayList<>(); tags = new ArrayList<>();
} }
public Tags(final List<Tag> tags) { public Tags(final List<Tag> tags) {
Collections.sort(tags, TagByKeyAndValueComparator.INSTANCE); Collections.sort(tags, TagByKeyAndValueComparator.INSTANCE);
this.tags = tags; this.tags = tags;
} }
public static Tags create(final List<Tag> tags) { public static Tags create(final List<Tag> tags) {
return new Tags(tags); return new Tags(tags);
} }
public static Tags create() { public static Tags create() {
return EMPTY; return EMPTY;
} }
public static Tags create(final int key, final int value) { public static Tags create(final int key, final int value) {
return TagsBuilder.create().add(key, value).build(); return TagsBuilder.create().add(key, value).build();
} }
public static Tags create(final int key1, final int value1, final int key2, final int value2) { public static Tags create(final int key1, final int value1, final int key2, final int value2) {
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).build(); final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).build();
return result; return result;
} }
public static Tags create(final int key1, final int value1, final int key2, final int value2, final int key3, public static Tags create(final int key1, final int value1, final int key2, final int value2, final int key3,
final int value3) { final int value3) {
final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).add(key3, value3).build(); final Tags result = TagsBuilder.create().add(key1, value1).add(key2, value2).add(key3, value3).build();
return result; return result;
} }
public static Tags createAndAddToDictionary(final String key, final String value) { public static Tags createAndAddToDictionary(final String key, final String value) {
return TagsBuilder.create().addAndAddToDictionary(key, value).build(); return TagsBuilder.create().addAndAddToDictionary(key, value).build();
} }
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2, public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2) { final String value2) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2) final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.build(); .build();
return result; return result;
} }
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2, public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3) { final String value2, final String key3, final String value3) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2) final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).build(); .addAndAddToDictionary(key3, value3).build();
return result; return result;
} }
public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2, public static Tags createAndAddToDictionary(final String key1, final String value1, final String key2,
final String value2, final String key3, final String value3, final String key4, final String value4) { final String value2, final String key3, final String value3, final String key4, final String value4) {
final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2) final Tags result = TagsBuilder.create().addAndAddToDictionary(key1, value1).addAndAddToDictionary(key2, value2)
.addAndAddToDictionary(key3, value3).addAndAddToDictionary(key4, value4).build(); .addAndAddToDictionary(key3, value3).addAndAddToDictionary(key4, value4).build();
return result; return result;
} }
public static Tags fromBytes(final byte[] bytes) { public static Tags fromBytes(final byte[] bytes) {
final List<Tag> result = new ArrayList<>(); final List<Tag> result = new ArrayList<>();
final LongList keyValuesAsLongs = VariableByteEncoder.decode(bytes); final LongList keyValuesAsLongs = VariableByteEncoder.decode(bytes);
for (int i = 0; i < keyValuesAsLongs.size(); i += 2) { for (int i = 0; i < keyValuesAsLongs.size(); i += 2) {
final long keyAsLong = keyValuesAsLongs.get(i); final long keyAsLong = keyValuesAsLongs.get(i);
final long valueAsLong = keyValuesAsLongs.get(i + 1); final long valueAsLong = keyValuesAsLongs.get(i + 1);
final int key = (int) keyAsLong; final int key = (int) keyAsLong;
final int value = (int) valueAsLong; final int value = (int) valueAsLong;
result.add(new Tag(key, value)); result.add(new Tag(key, value));
} }
return new Tags(result); return new Tags(result);
} }
public byte[] toBytes() { public byte[] toBytes() {
final byte[] result; final byte[] result;
if (tags.size() > 0) { if (tags.size() > 0) {
final LongList keyValuesAsLongs = new LongList(tags.size() * 2); final LongList keyValuesAsLongs = new LongList(tags.size() * 2);
for (final Tag tag : tags) { for (final Tag tag : tags) {
final long keyAsLong = tag.getKey(); final long keyAsLong = tag.getKey();
final long valueAsLong = tag.getValue(); final long valueAsLong = tag.getValue();
keyValuesAsLongs.add(keyAsLong); keyValuesAsLongs.add(keyAsLong);
keyValuesAsLongs.add(valueAsLong); keyValuesAsLongs.add(valueAsLong);
} }
result = VariableByteEncoder.encode(keyValuesAsLongs); result = VariableByteEncoder.encode(keyValuesAsLongs);
} else { } else {
result = EMPTY_BYTES; result = EMPTY_BYTES;
} }
return result; return result;
} }
@Override @Override
public int compareTo(final Tags o) { public int compareTo(final Tags o) {
if (tags.size() != o.tags.size()) { if (tags.size() != o.tags.size()) {
return tags.size() - o.tags.size(); return tags.size() - o.tags.size();
} else { } else {
for (int i = 0; i < tags.size(); i++) { for (int i = 0; i < tags.size(); i++) {
final int compareResult = tags.get(i).compareTo(o.tags.get(i)); final int compareResult = tags.get(i).compareTo(o.tags.get(i));
if (compareResult != 0) { if (compareResult != 0) {
return compareResult; return compareResult;
} }
} }
} }
return 0; return 0;
} }
public String getValue(final String key) { public String getValue(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0); final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE); final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) { if (index >= 0) {
final Tag tag = tags.get(index); final Tag tag = tags.get(index);
return STRING_COMPRESSOR.get(tag.getValue()); return STRING_COMPRESSOR.get(tag.getValue());
} }
return null; return null;
} }
public int getValueAsInt(final String key) { public int getValueAsInt(final String key) {
final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0); final Tag needle = new Tag(STRING_COMPRESSOR.put(key), 0);
final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE); final int index = Collections.binarySearch(tags, needle, TagByKeyComparator.INSTANCE);
if (index >= 0) { if (index >= 0) {
final Tag tag = tags.get(index); final Tag tag = tags.get(index);
return tag.getValue(); return tag.getValue();
} }
return -1; return -1;
} }
public Set<String> getKeys() { public Set<String> getKeys() {
final TreeSet<String> result = new TreeSet<>(); final TreeSet<String> result = new TreeSet<>();
for (final Tag tag : tags) { for (final Tag tag : tags) {
result.add(STRING_COMPRESSOR.get(tag.getKey())); result.add(STRING_COMPRESSOR.get(tag.getKey()));
} }
return result; return result;
} }
public IntList getKeysAsInt() { public IntList getKeysAsInt() {
final IntList result = new IntList(); final IntList result = new IntList();
for (final Tag tag : tags) { for (final Tag tag : tags) {
result.add(tag.getKey()); result.add(tag.getKey());
} }
return result; return result;
} }
public List<Tag> toTags() { public List<Tag> toTags() {
return Collections.unmodifiableList(tags); return Collections.unmodifiableList(tags);
} }
public void forEach(final BiConsumer<String, String> keyValueConsumer) { public void forEach(final BiConsumer<String, String> keyValueConsumer) {
for (final Tag tag : tags) { for (final Tag tag : tags) {
final String key = STRING_COMPRESSOR.get(tag.getKey()); final String key = STRING_COMPRESSOR.get(tag.getKey());
final String value = STRING_COMPRESSOR.get(tag.getValue()); final String value = STRING_COMPRESSOR.get(tag.getValue());
keyValueConsumer.accept(key, value); keyValueConsumer.accept(key, value);
} }
} }
public Tags mapTags(final Function<Tag, Tag> tagMapFuntion) { public Tags mapTags(final Function<Tag, Tag> tagMapFuntion) {
final List<Tag> mappedTags = new ArrayList<>(tags.size()); final List<Tag> mappedTags = new ArrayList<>(tags.size());
for (final Tag tag : tags) { for (final Tag tag : tags) {
mappedTags.add(tagMapFuntion.apply(tag)); mappedTags.add(tagMapFuntion.apply(tag));
} }
return Tags.create(mappedTags); return Tags.create(mappedTags);
} }
@Override @Override
public String toString() { public String toString() {
return String.valueOf(tags); return String.valueOf(tags);
} }
public String toCsv() { public String toCsv() {
final List<String> tagsAsStrings = new ArrayList<>(); final List<String> tagsAsStrings = new ArrayList<>();
for (final Tag tag : tags) { for (final Tag tag : tags) {
tagsAsStrings.add(tag.getKeyAsString() + "=" + tag.getValueAsString()); tagsAsStrings.add(tag.getKeyAsString() + "=" + tag.getValueAsString());
} }
return String.join(",", tagsAsStrings); return String.join(",", tagsAsStrings);
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + ((tags == null) ? 0 : tags.hashCode()); result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final Tags other = (Tags) obj; final Tags other = (Tags) obj;
if (tags == null) { if (tags == null) {
if (other.tags != null) if (other.tags != null)
return false; return false;
} else if (!tags.equals(other.tags)) } else if (!tags.equals(other.tags))
return false; return false;
return true; return true;
} }
public Tags subset(final List<String> groupByFields) { public Tags subset(final List<String> groupByFields) {
final TagsBuilder result = TagsBuilder.create(); final TagsBuilder result = TagsBuilder.create();
for (final String field : groupByFields) { for (final String field : groupByFields) {
final int value = getValueAsInt(field); final int value = getValueAsInt(field);
if (value >= 0) { if (value >= 0) {
final int fieldAsInt = STRING_COMPRESSOR.getIfPresent(field); final int fieldAsInt = STRING_COMPRESSOR.getIfPresent(field);
result.add(fieldAsInt, value); result.add(fieldAsInt, value);
} }
} }
return result.build(); return result.build();
} }
public boolean isEmpty() { public boolean isEmpty() {
return tags.isEmpty(); return tags.isEmpty();
} }
/** /**
* @return User facing readable representation * @return User facing readable representation
*/ */
public String asString() { public String asString() {
final StringBuilder result = new StringBuilder(); final StringBuilder result = new StringBuilder();
for (final Tag tag : tags) { for (final Tag tag : tags) {
if (result.length() > 0) { if (result.length() > 0) {
result.append(", "); result.append(", ");
} }
result.append(STRING_COMPRESSOR.get(tag.getKey())); result.append(STRING_COMPRESSOR.get(tag.getKey()));
result.append("="); result.append("=");
result.append(STRING_COMPRESSOR.get(tag.getValue())); result.append(STRING_COMPRESSOR.get(tag.getValue()));
} }
return result.toString(); return result.toString();
} }
} }

42
pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java
View File

@@ -5,30 +5,30 @@ import java.util.List;
public class TagsBuilder { public class TagsBuilder {
final List<Tag> tags = new ArrayList<>(); final List<Tag> tags = new ArrayList<>();
public static TagsBuilder create() { public static TagsBuilder create() {
return new TagsBuilder(); return new TagsBuilder();
} }
public TagsBuilder add(final int key, final int value) { public TagsBuilder add(final int key, final int value) {
tags.add(new Tag(key, value)); tags.add(new Tag(key, value));
return this; return this;
} }
public TagsBuilder add(final String key, final String value) { public TagsBuilder add(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.getIfPresent(key); final int keyAsInt = Tags.STRING_COMPRESSOR.getIfPresent(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.getIfPresent(value); final int valueAsInt = Tags.STRING_COMPRESSOR.getIfPresent(value);
return add(keyAsInt, valueAsInt); return add(keyAsInt, valueAsInt);
} }
public TagsBuilder addAndAddToDictionary(final String key, final String value) { public TagsBuilder addAndAddToDictionary(final String key, final String value) {
final int keyAsInt = Tags.STRING_COMPRESSOR.put(key); final int keyAsInt = Tags.STRING_COMPRESSOR.put(key);
final int valueAsInt = Tags.STRING_COMPRESSOR.put(value); final int valueAsInt = Tags.STRING_COMPRESSOR.put(value);
return add(keyAsInt, valueAsInt); return add(keyAsInt, valueAsInt);
} }
public Tags build() { public Tags build() {
return Tags.create(tags); return Tags.create(tags);
} }
} }

288
pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java
View File

@@ -31,182 +31,182 @@ import java.util.regex.Pattern;
* retrievals. * retrievals.
*/ */
public class UniqueStringIntegerPairs { public class UniqueStringIntegerPairs {
private static final String SEPARATOR = "\t"; private static final String SEPARATOR = "\t";
private static final boolean APPEND = true; private static final boolean APPEND = true;
private static final class ByteArray implements Comparable<ByteArray> { private static final class ByteArray implements Comparable<ByteArray> {
private final byte[] array; private final byte[] array;
private final int start; private final int start;
private final int endExclusive; private final int endExclusive;
public ByteArray(final byte[] array, final int start, final int endExclusive) { public ByteArray(final byte[] array, final int start, final int endExclusive) {
super(); super();
this.array = array; this.array = array;
this.start = start; this.start = start;
this.endExclusive = endExclusive; this.endExclusive = endExclusive;
} }
public ByteArray(final byte[] bytes) { public ByteArray(final byte[] bytes) {
this.array = bytes; this.array = bytes;
this.start = 0; this.start = 0;
this.endExclusive = bytes.length; this.endExclusive = bytes.length;
} }
// custom hashcode! // custom hashcode!
@Override @Override
public int hashCode() { public int hashCode() {
int result = 1; int result = 1;
final byte[] a = array; final byte[] a = array;
final int end = endExclusive; final int end = endExclusive;
for (int i = start; i < end; i++) { for (int i = start; i < end; i++) {
result = 31 * result + a[i]; result = 31 * result + a[i];
} }
return result; return result;
} }
// custom equals! // custom equals!
@Override @Override
public boolean equals(final Object obj) { public boolean equals(final Object obj) {
final ByteArray other = (ByteArray) obj; final ByteArray other = (ByteArray) obj;
if (!Arrays.equals(array, start, endExclusive, other.array, other.start, other.endExclusive)) if (!Arrays.equals(array, start, endExclusive, other.array, other.start, other.endExclusive))
return false; return false;
return true; return true;
} }
@Override @Override
public int compareTo(final ByteArray o) { public int compareTo(final ByteArray o) {
return Arrays.compare(array, start, endExclusive, o.array, o.start, o.endExclusive); return Arrays.compare(array, start, endExclusive, o.array, o.start, o.endExclusive);
} }
} }
/** /**
* Maps a string to an integer. E.g. "myLongValue" -> 123 * Maps a string to an integer. E.g. "myLongValue" -> 123
*/ */
private final Map<String, Integer> stringToInt = new HashMap<>(); private final Map<String, Integer> stringToInt = new HashMap<>();
private final Map<ByteArray, Integer> bytesToInt = new HashMap<>(); private final Map<ByteArray, Integer> bytesToInt = new HashMap<>();
/** /**
* Maps an integer to a string. E.g. 123 -> "myLongValue" * Maps an integer to a string. E.g. 123 -> "myLongValue"
*/ */
private final List<String> intToString = new ArrayList<>(); private final List<String> intToString = new ArrayList<>();
private final Path file; private final Path file;
public UniqueStringIntegerPairs() { public UniqueStringIntegerPairs() {
this(null); this(null);
} }
public UniqueStringIntegerPairs(final Path file) { public UniqueStringIntegerPairs(final Path file) {
this.file = file; this.file = file;
if (file != null) { if (file != null) {
init(file); init(file);
} }
} }
private void init(final Path file) throws RuntimeIOException { private void init(final Path file) throws RuntimeIOException {
try { try {
Files.createDirectories(file.getParent()); Files.createDirectories(file.getParent());
if (!Files.exists(file)) { if (!Files.exists(file)) {
Files.createFile(file); Files.createFile(file);
} }
try (final BufferedReader reader = new BufferedReader( try (final BufferedReader reader = new BufferedReader(
new InputStreamReader(new FileInputStream(file.toFile()), StandardCharsets.UTF_8))) { new InputStreamReader(new FileInputStream(file.toFile()), StandardCharsets.UTF_8))) {
String line; String line;
while ((line = reader.readLine()) != null) { while ((line = reader.readLine()) != null) {
final String[] tokens = line.split(Pattern.quote(SEPARATOR)); final String[] tokens = line.split(Pattern.quote(SEPARATOR));
if (tokens.length == 2) { if (tokens.length == 2) {
final String string = tokens[0]; final String string = tokens[0];
final int integer = Integer.parseInt(tokens[1]); final int integer = Integer.parseInt(tokens[1]);
intToStringPut(integer, string); intToStringPut(integer, string);
stringToInt.put(string, integer); stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer); bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
} }
} }
} }
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
private void intToStringPut(final int value, final String string) { private void intToStringPut(final int value, final String string) {
if (intToString.size() <= value) { if (intToString.size() <= value) {
// list is not long enough -> grow list // list is not long enough -> grow list
while (intToString.size() <= value) { while (intToString.size() <= value) {
intToString.add(null); intToString.add(null);
} }
} }
intToString.set(value, string); intToString.set(value, string);
} }
void put(final String string, final int integer) { void put(final String string, final int integer) {
if (stringToInt.containsKey(string) || (intToString.size() > integer && intToString.get(integer) != null)) { if (stringToInt.containsKey(string) || (intToString.size() > integer && intToString.get(integer) != null)) {
throw new IllegalArgumentException("Unique key constraint violation for (" + string + ", " + integer + ")"); throw new IllegalArgumentException("Unique key constraint violation for (" + string + ", " + integer + ")");
} }
if (file != null) { if (file != null) {
try (final Writer writer = new OutputStreamWriter(new FileOutputStream(file.toFile(), APPEND), try (final Writer writer = new OutputStreamWriter(new FileOutputStream(file.toFile(), APPEND),
StandardCharsets.UTF_8)) { StandardCharsets.UTF_8)) {
writer.write(string + SEPARATOR + integer + "\n"); writer.write(string + SEPARATOR + integer + "\n");
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} }
} }
intToStringPut(integer, string); intToStringPut(integer, string);
stringToInt.put(string, integer); stringToInt.put(string, integer);
bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer); bytesToInt.put(new ByteArray(string.getBytes(StandardCharsets.UTF_8)), integer);
} }
public Integer get(final String string) { public Integer get(final String string) {
return stringToInt.get(string); return stringToInt.get(string);
} }
public String getKey(final int second) { public String getKey(final int second) {
return intToString.get(second); return intToString.get(second);
} }
public Integer getHighestInteger() { public Integer getHighestInteger() {
return intToString.size() == 0 ? -1 : intToString.size() - 1; return intToString.size() == 0 ? -1 : intToString.size() - 1;
} }
public Integer computeIfAbsent(final String string, final Function<String, Integer> mappingFunction) { public Integer computeIfAbsent(final String string, final Function<String, Integer> mappingFunction) {
if (!stringToInt.containsKey(string)) { if (!stringToInt.containsKey(string)) {
synchronized (stringToInt) { synchronized (stringToInt) {
if (!stringToInt.containsKey(string)) { if (!stringToInt.containsKey(string)) {
final Integer second = mappingFunction.apply(string); final Integer second = mappingFunction.apply(string);
put(string, second); put(string, second);
} }
} }
} }
return stringToInt.get(string); return stringToInt.get(string);
} }
public Integer computeIfAbsent(final byte[] bytes, final int start, final int endExclusive) { public Integer computeIfAbsent(final byte[] bytes, final int start, final int endExclusive) {
final ByteArray byteArray = new ByteArray(bytes, start, endExclusive); final ByteArray byteArray = new ByteArray(bytes, start, endExclusive);
Integer result = bytesToInt.get(byteArray); Integer result = bytesToInt.get(byteArray);
if (result == null) { if (result == null) {
synchronized (stringToInt) { synchronized (stringToInt) {
if (!bytesToInt.containsKey(byteArray)) { if (!bytesToInt.containsKey(byteArray)) {
final String string = new String(bytes, start, endExclusive - start, StandardCharsets.UTF_8); final String string = new String(bytes, start, endExclusive - start, StandardCharsets.UTF_8);
final Integer integer = intToString.size(); final Integer integer = intToString.size();
put(string, integer); put(string, integer);
} }
result = bytesToInt.get(byteArray); result = bytesToInt.get(byteArray);
} }
} }
return result; return result;
} }
} }

210
pdb-api/src/test/java/org/lucares/memory/MemoryScale.java
View File

@@ -14,133 +14,133 @@ import org.lucares.pdb.api.UniqueStringIntegerPairs;
public class MemoryScale { public class MemoryScale {
public static final String A = "A"; public static final String A = "A";
public static void main(final String[] args) { public static void main(final String[] args) {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs()); Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
scale("singleTag"); scale("singleTag");
scale("tags0"); scale("tags0");
scale("tags1"); scale("tags1");
scale("tags2"); scale("tags2");
scale("tags6"); scale("tags6");
} }
private static void scale(final String what) { private static void scale(final String what) {
System.out.println("start: " + what); System.out.println("start: " + what);
// warmup of classes // warmup of classes
getUsedMemory(); getUsedMemory();
Object handle = createObject(what); Object handle = createObject(what);
handle = null; handle = null;
runGc(); runGc();
final long memoryBefore = getUsedMemory(); final long memoryBefore = getUsedMemory();
handle = createObject(what); handle = createObject(what);
runGc(); runGc();
final long memoryAfter = getUsedMemory(); final long memoryAfter = getUsedMemory();
System.out.println(what + ": used memory: " + (memoryAfter - memoryBefore)); System.out.println(what + ": used memory: " + (memoryAfter - memoryBefore));
handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler
} }
private static Object createObject(final String what) { private static Object createObject(final String what) {
switch (what) { switch (what) {
case "singleTag": case "singleTag":
return createTag(); return createTag();
case "tags0": case "tags0":
return createTags0(); return createTags0();
case "tags1": case "tags1":
return createTags1(); return createTags1();
case "tags2": case "tags2":
return createTags2(); return createTags2();
case "tags6": case "tags6":
return createTags6(); return createTags6();
case "string": case "string":
return createString(); return createString();
case "linkedHashMap": case "linkedHashMap":
return createLinkedHashMap(); return createLinkedHashMap();
case "path": case "path":
return createPath("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb"); return createPath("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsString": case "pathAsString":
return createPathAsString("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb"); return createPathAsString("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
case "pathAsUtf8": case "pathAsUtf8":
return createPathAsUtf8("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb"); return createPathAsUtf8("C:\\pdb\\dataNew\\storage\\0\\4\\3n-5k_0-5l_2-1L_4-4n_3w-5h_6-7$.pdb");
default: default:
return null; return null;
} }
} }
private static Object createTag() { private static Object createTag() {
return new Tag("", ""); return new Tag("", "");
} }
private static Object createTags0() { private static Object createTags0() {
return new Tags(); return new Tags();
} }
private static Object createTags1() { private static Object createTags1() {
return Tags.createAndAddToDictionary("k1", "v1"); return Tags.createAndAddToDictionary("k1", "v1");
} }
private static Object createTags2() { private static Object createTags2() {
return Tags.createAndAddToDictionary("k1", "v1", "k2", "v2"); return Tags.createAndAddToDictionary("k1", "v1", "k2", "v2");
} }
private static Object createTags6() { private static Object createTags6() {
TagsBuilder result = TagsBuilder.create(); TagsBuilder result = TagsBuilder.create();
result = result.add("k1", "v1"); result = result.add("k1", "v1");
result = result.add("k2", "v2"); result = result.add("k2", "v2");
result = result.add("k3", "v3"); result = result.add("k3", "v3");
result = result.add("k4", "v4"); result = result.add("k4", "v4");
result = result.add("k5", "v5"); result = result.add("k5", "v5");
result = result.add("k6", "v6"); result = result.add("k6", "v6");
return result.build(); return result.build();
} }
private static Object createPathAsUtf8(final String string) { private static Object createPathAsUtf8(final String string) {
return string.getBytes(StandardCharsets.UTF_8); return string.getBytes(StandardCharsets.UTF_8);
} }
private static String createPathAsString(final String string) { private static String createPathAsString(final String string) {
return string.replace("C", "c"); return string.replace("C", "c");
} }
private static Path createPath(final String string) { private static Path createPath(final String string) {
return Paths.get(string); return Paths.get(string);
} }
private static String createString() { private static String createString() {
final int i = 0; final int i = 0;
return "" + i; return "" + i;
} }
private static Object createLinkedHashMap() { private static Object createLinkedHashMap() {
final Map<String, String> map = new LinkedHashMap<>(); final Map<String, String> map = new LinkedHashMap<>();
map.put("A", "A"); map.put("A", "A");
for (int i = 0; i < 0; i++) { for (int i = 0; i < 0; i++) {
map.put("" + i, "" + i); map.put("" + i, "" + i);
} }
return map; return map;
} }
private static void runGc() { private static void runGc() {
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
System.gc(); System.gc();
try { try {
Thread.sleep(100); Thread.sleep(100);
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
} }
} }
private static long getUsedMemory() { private static long getUsedMemory() {
return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
} }
} }

38
pdb-api/src/test/java/org/lucares/pdb/api/DateTimeRangeTest.java
View File

@@ -11,27 +11,27 @@ import org.testng.annotations.DataProvider;
import org.testng.annotations.Test; import org.testng.annotations.Test;
public class DateTimeRangeTest { public class DateTimeRangeTest {
@DataProvider @DataProvider
Object[][] providerIntersect() { Object[][] providerIntersect() {
final List<Object[]> result = new ArrayList<>(); final List<Object[]> result = new ArrayList<>();
final OffsetDateTime a = Instant.ofEpochMilli(1000).atOffset(ZoneOffset.UTC); final OffsetDateTime a = Instant.ofEpochMilli(1000).atOffset(ZoneOffset.UTC);
final OffsetDateTime b = Instant.ofEpochMilli(2000).atOffset(ZoneOffset.UTC); final OffsetDateTime b = Instant.ofEpochMilli(2000).atOffset(ZoneOffset.UTC);
final OffsetDateTime c = Instant.ofEpochMilli(3000).atOffset(ZoneOffset.UTC); final OffsetDateTime c = Instant.ofEpochMilli(3000).atOffset(ZoneOffset.UTC);
final OffsetDateTime d = Instant.ofEpochMilli(4000).atOffset(ZoneOffset.UTC); final OffsetDateTime d = Instant.ofEpochMilli(4000).atOffset(ZoneOffset.UTC);
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(c, d), false }); result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(c, d), false });
result.add(new Object[] { new DateTimeRange(a, c), new DateTimeRange(b, d), true }); result.add(new Object[] { new DateTimeRange(a, c), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true }); result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true }); result.add(new Object[] { new DateTimeRange(a, d), new DateTimeRange(b, d), true });
result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(b, d), true }); result.add(new Object[] { new DateTimeRange(a, b), new DateTimeRange(b, d), true });
return result.toArray(new Object[result.size()][]); return result.toArray(new Object[result.size()][]);
} }
@Test(dataProvider = "providerIntersect") @Test(dataProvider = "providerIntersect")
public void testIntersect(final DateTimeRange a, final DateTimeRange b, final boolean expected) throws Exception { public void testIntersect(final DateTimeRange a, final DateTimeRange b, final boolean expected) throws Exception {
Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b); Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b);
Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b); Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b);
} }
} }

88
pdb-api/src/test/java/org/lucares/pdb/api/StringCompressorTest.java
View File

@@ -18,63 +18,63 @@ import org.testng.annotations.Test;
@Test @Test
public class StringCompressorTest { public class StringCompressorTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void testKeyCompressorRoundtrip() throws Exception { public void testKeyCompressorRoundtrip() throws Exception {
final StringCompressor keyValueCompressor = StringCompressor.create(dataDirectory.resolve("key.csv")); final StringCompressor keyValueCompressor = StringCompressor.create(dataDirectory.resolve("key.csv"));
final String value = "foo"; final String value = "foo";
final Integer intFoo = keyValueCompressor.put(value); final Integer intFoo = keyValueCompressor.put(value);
final String actual = keyValueCompressor.get(intFoo); final String actual = keyValueCompressor.get(intFoo);
Assert.assertEquals(actual, value); Assert.assertEquals(actual, value);
} }
public void testKeyCompressorInitialization() throws Exception { public void testKeyCompressorInitialization() throws Exception {
final Path database = dataDirectory.resolve("key.csv"); final Path database = dataDirectory.resolve("key.csv");
final String value = "foo"; final String value = "foo";
{ {
final StringCompressor keyValueCompressor = StringCompressor.create(database); final StringCompressor keyValueCompressor = StringCompressor.create(database);
keyValueCompressor.put(value); keyValueCompressor.put(value);
} }
{ {
final StringCompressor keyValueCompressor = StringCompressor.create(database); final StringCompressor keyValueCompressor = StringCompressor.create(database);
keyValueCompressor.get(0); keyValueCompressor.get(0);
} }
} }
@Test(invocationCount = 1) @Test(invocationCount = 1)
public void testPutConcurrently() throws InterruptedException, ExecutionException { public void testPutConcurrently() throws InterruptedException, ExecutionException {
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(); final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs();
final StringCompressor stringCompressor = new StringCompressor(usip); final StringCompressor stringCompressor = new StringCompressor(usip);
final ExecutorService pool = Executors.newCachedThreadPool(); final ExecutorService pool = Executors.newCachedThreadPool();
final int numEntries = 1000; final int numEntries = 1000;
final Future<List<String>> future1 = pool.submit(new StringInserter(stringCompressor, numEntries)); final Future<List<String>> future1 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future2 = pool.submit(new StringInserter(stringCompressor, numEntries)); final Future<List<String>> future2 = pool.submit(new StringInserter(stringCompressor, numEntries));
final Future<List<String>> future3 = pool.submit(new StringInserter(stringCompressor, numEntries)); final Future<List<String>> future3 = pool.submit(new StringInserter(stringCompressor, numEntries));
future1.get(); future1.get();
future2.get(); future2.get();
future3.get(); future3.get();
pool.shutdown(); pool.shutdown();
pool.awaitTermination(1, TimeUnit.MILLISECONDS); pool.awaitTermination(1, TimeUnit.MILLISECONDS);
Assert.assertEquals((int) usip.getHighestInteger(), 3 * numEntries - 1); Assert.assertEquals((int) usip.getHighestInteger(), 3 * numEntries - 1);
} }
} }

32
pdb-api/src/test/java/org/lucares/pdb/api/StringInserter.java
View File

@@ -7,23 +7,23 @@ import java.util.concurrent.Callable;
final class StringInserter implements Callable<List<String>> { final class StringInserter implements Callable<List<String>> {
private final StringCompressor stringCompressor; private final StringCompressor stringCompressor;
private final int numEntries; private final int numEntries;
public StringInserter(final StringCompressor stringCompressor, final int numEntries) { public StringInserter(final StringCompressor stringCompressor, final int numEntries) {
this.stringCompressor = stringCompressor; this.stringCompressor = stringCompressor;
this.numEntries = numEntries; this.numEntries = numEntries;
} }
@Override @Override
public List<String> call() throws Exception { public List<String> call() throws Exception {
final List<String> result = new ArrayList<>(); final List<String> result = new ArrayList<>();
for (int i = 0; i < numEntries; i++) { for (int i = 0; i < numEntries; i++) {
final String s = UUID.randomUUID().toString(); final String s = UUID.randomUUID().toString();
stringCompressor.put(s); stringCompressor.put(s);
result.add(s); result.add(s);
} }
return result; return result;
} }
} }

96
pdb-api/src/test/java/org/lucares/pdb/api/UniqueStringIntegerPairsTest.java
View File

@@ -13,62 +13,62 @@ import org.testng.annotations.Test;
@Test @Test
public class UniqueStringIntegerPairsTest { public class UniqueStringIntegerPairsTest {
private Path dataDirectory; private Path dataDirectory;
@BeforeMethod @BeforeMethod
public void beforeMethod() throws IOException { public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
} }
@AfterMethod @AfterMethod
public void afterMethod() throws IOException { public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
} }
public void testPutGet() throws Exception { public void testPutGet() throws Exception {
final Path database = dataDirectory.resolve("key.csv"); final Path database = dataDirectory.resolve("key.csv");
final String first = "key1"; final String first = "key1";
final Integer second = 1; final Integer second = 1;
{ {
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database); final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
usip.put(first, second); usip.put(first, second);
Assert.assertEquals(usip.get(first), second); Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first); Assert.assertEquals(usip.getKey(second), first);
} }
{ {
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database); final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
Assert.assertEquals(usip.get(first), second); Assert.assertEquals(usip.get(first), second);
Assert.assertEquals(usip.getKey(second), first); Assert.assertEquals(usip.getKey(second), first);
} }
} }
public void testUniqueKeyContstraint() throws Exception { public void testUniqueKeyContstraint() throws Exception {
final Path database = dataDirectory.resolve("key.csv"); final Path database = dataDirectory.resolve("key.csv");
final String first = "key1"; final String first = "key1";
final Integer second = 1; final Integer second = 1;
final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database); final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
usip.put(first, second); usip.put(first, second);
try { try {
// cannot add another pair with the first key // cannot add another pair with the first key
final int another = second + 1; final int another = second + 1;
usip.put(first, another); usip.put(first, another);
Assert.fail("expected an IllegalArgumentException"); Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) { } catch (final IllegalArgumentException e) {
// expected // expected
} }
try { try {
// cannot add another pair with the same second value // cannot add another pair with the same second value
final String another = first + 1; final String another = first + 1;
usip.put(another, second); usip.put(another, second);
Assert.fail("expected an IllegalArgumentException"); Assert.fail("expected an IllegalArgumentException");
} catch (final IllegalArgumentException e) { } catch (final IllegalArgumentException e) {
// expected // expected
} }
} }
} }

23
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java
View File

@@ -1,17 +1,18 @@
package org.lucares.pdb.plot.api; package org.lucares.pdb.plot.api;
/** /**
* Note: The order in this enum defines the order in which the aggregates are drawn. * Note: The order in this enum defines the order in which the aggregates are
* drawn.
*/ */
public enum Aggregate { public enum Aggregate {
PARALLEL, PARALLEL,
SCATTER, SCATTER,
/** /**
* Empirical cumulative distribution functions * Empirical cumulative distribution functions
* *
* @see https://serialmentor.com/dataviz/ecdf-qq.html * @see https://serialmentor.com/dataviz/ecdf-qq.html
*/ */
CUM_DISTRIBUTION, CUM_DISTRIBUTION,
} }

77
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandler.java
View File

@@ -13,52 +13,53 @@ import org.lucares.recommind.logs.LineStyle;
public abstract class AggregateHandler implements Appender { public abstract class AggregateHandler implements Appender {
private GnuplotAxis xAxis = GnuplotAxis.X1; private GnuplotAxis xAxis = GnuplotAxis.X1;
private GnuplotAxis yAxis = GnuplotAxis.Y1;
public GnuplotAxis getxAxis() {
return xAxis;
}
public void updateAxis(GnuplotAxis axis) { private GnuplotAxis yAxis = GnuplotAxis.Y1;
switch (axis) {
case X1: public GnuplotAxis getxAxis() {
case X2: return xAxis;
this.xAxis = axis;
break;
case Y1:
case Y2:
this.yAxis = axis;
break;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
} }
}
public GnuplotAxis getyAxis() { public void updateAxis(final GnuplotAxis axis) {
return yAxis; switch (axis) {
} case X1:
case X2:
this.xAxis = axis;
break;
case Y1:
case Y2:
this.yAxis = axis;
break;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
protected String gnuplotXYAxis() { public GnuplotAxis getyAxis() {
return xAxis.getAxisNameForPlots()+yAxis.getAxisNameForPlots(); return yAxis;
} }
abstract Type getAxisType(GnuplotAxis axis);
abstract Aggregate getAggregateType(); protected String gnuplotXYAxis() {
return xAxis.getAxisNameForPlots() + yAxis.getAxisNameForPlots();
}
abstract AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries); abstract Type getAxisType(GnuplotAxis axis);
abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries); abstract Aggregate getAggregateType();
abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Optional<String> title); abstract AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli, abstract AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries);
long toEpochMilli);
protected String gnuplotTitle(Optional<String> title) { abstract void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title);
return title.isPresent() ? "title '" + title.get() + "'" : "notitle";
} abstract CustomAggregator createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli);
protected String gnuplotTitle(final Optional<String> title) {
return title.isPresent() ? "title '" + title.get() + "'" : "notitle";
}
} }

166
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregateHandlerCollection.java
View File

@@ -17,91 +17,97 @@ import org.lucares.utils.CollectionUtils;
import org.lucares.utils.Preconditions; import org.lucares.utils.Preconditions;
public class AggregateHandlerCollection { public class AggregateHandlerCollection {
private static final Comparator<AggregateHandler> PLOTTING_ORDER = Comparator.comparing(AggregateHandler::getAggregateType); private static final Comparator<AggregateHandler> PLOTTING_ORDER = Comparator
.comparing(AggregateHandler::getAggregateType);
private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
public void add(AggregateHandler aggregateHandler) { private final List<AggregateHandler> aggregateHandlers = new ArrayList<>();
aggregateHandlers.add(aggregateHandler);
}
public void updateAxisForHandlers() {
updateAxisForHandlers(GnuplotAxis.X1);
updateAxisForHandlers(GnuplotAxis.Y1);
}
private void updateAxisForHandlers(GnuplotAxis axis) { public void add(final AggregateHandler aggregateHandler) {
final EnumSet<Type> result = EnumSet.noneOf(Type.class); aggregateHandlers.add(aggregateHandler);
for (AggregateHandler handler : aggregateHandlers) { }
final Type type = handler.getAxisType(axis);
public void updateAxisForHandlers() {
if (result.isEmpty()) { updateAxisForHandlers(GnuplotAxis.X1);
result.add(type); updateAxisForHandlers(GnuplotAxis.Y1);
}else { }
final boolean containsType = result.contains(type);
if (containsType) { private void updateAxisForHandlers(final GnuplotAxis axis) {
// already has an axis of this type final EnumSet<Type> result = EnumSet.noneOf(Type.class);
// TODO merge axis definitions and use the greater values for: range, ticsIncrement for (final AggregateHandler handler : aggregateHandlers) {
} else{ final Type type = handler.getAxisType(axis);
Preconditions.checkSmaller(result.size(), 2, "at most two different axis are supported");
final GnuplotAxis mirrorAxis = axis.mirrorAxis(); if (result.isEmpty()) {
handler.updateAxis(mirrorAxis); result.add(type);
result.add(type); } else {
final boolean containsType = result.contains(type);
if (containsType) {
// already has an axis of this type
// TODO merge axis definitions and use the greater values for: range,
// ticsIncrement
} else {
Preconditions.checkSmaller(result.size(), 2, "at most two different axis are supported");
final GnuplotAxis mirrorAxis = axis.mirrorAxis();
handler.updateAxis(mirrorAxis);
result.add(type);
}
}
} }
}
}
}
public List<AxisSettings> getXAxisDefinitions(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
final List<AxisSettings> result = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
AxisSettings axis = handler.createXAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public List<AxisSettings> getYAxisDefinitions(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
List<AxisSettings> result = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
final AxisSettings axis = handler.createYAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public AggregatorCollection createCustomAggregator(Path tmpDir, PlotSettings plotSettings, long fromEpochMilli,
long toEpochMilli) {
final List<CustomAggregator> aggregators = new ArrayList<>();
for (AggregateHandler handler : aggregateHandlers) {
final CustomAggregator aggregator = handler.createCustomAggregator(tmpDir, plotSettings, fromEpochMilli, toEpochMilli);
if (aggregator != null) {
aggregators.add(aggregator);
}
} }
return new AggregatorCollection(aggregators); public List<AxisSettings> getXAxisDefinitions(final GnuplotSettings settings,
} final Collection<DataSeries> dataSeries) {
final List<AxisSettings> result = new ArrayList<>();
public void addPlots(StringBuilder result, Collection<DataSeries> dataSeries) { for (final AggregateHandler handler : aggregateHandlers) {
final AxisSettings axis = handler.createXAxisSettings(settings, dataSeries);
boolean first = true; result.add(axis);
final List<AggregateHandler> handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers, PLOTTING_ORDER); }
for (AggregateHandler handler : handlersInPlottingOrder) { return result;
}
for (DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty(); public List<AxisSettings> getYAxisDefinitions(final GnuplotSettings settings,
final Collection<DataSeries> dataSeries) {
Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData().get(handler.getAggregateType()); final List<AxisSettings> result = new ArrayList<>();
if(aggregatedData.isPresent()) { for (final AggregateHandler handler : aggregateHandlers) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title); final AxisSettings axis = handler.createYAxisSettings(settings, dataSeries);
result.add(axis);
}
return result;
}
public AggregatorCollection createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) {
final List<CustomAggregator> aggregators = new ArrayList<>();
for (final AggregateHandler handler : aggregateHandlers) {
final CustomAggregator aggregator = handler.createCustomAggregator(tmpDir, plotSettings, fromEpochMilli,
toEpochMilli);
if (aggregator != null) {
aggregators.add(aggregator);
}
}
return new AggregatorCollection(aggregators);
}
public void addPlots(final StringBuilder result, final Collection<DataSeries> dataSeries) {
boolean first = true;
final List<AggregateHandler> handlersInPlottingOrder = CollectionUtils.copySort(aggregateHandlers,
PLOTTING_ORDER);
for (final AggregateHandler handler : handlersInPlottingOrder) {
for (final DataSeries dataSerie : dataSeries) {
final Optional<String> title = first ? Optional.of(dataSerie.getTitle()) : Optional.empty();
final Optional<AggregatedData> aggregatedData = dataSerie.getAggregatedData()
.get(handler.getAggregateType());
if (aggregatedData.isPresent()) {
handler.addPlot(result, aggregatedData.get(), dataSerie.getStyle(), title);
}
}
first = false;
} }
}
first = false;
} }
}
} }

24
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedData.java
View File

@@ -3,19 +3,19 @@ package org.lucares.pdb.plot.api;
import java.io.File; import java.io.File;
public class AggregatedData { public class AggregatedData {
private final String label; private final String label;
private final File dataFile; private final File dataFile;
public AggregatedData(final String label, final File dataFile) { public AggregatedData(final String label, final File dataFile) {
this.label = label; this.label = label;
this.dataFile = dataFile; this.dataFile = dataFile;
} }
public String getLabel() { public String getLabel() {
return label; return label;
} }
public File getDataFile() { public File getDataFile() {
return dataFile; return dataFile;
} }
} }

24
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataCollection.java
View File

@@ -4,19 +4,19 @@ import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Optional; import java.util.Optional;
public class AggregatedDataCollection implements Iterable<AggregatedData>{ public class AggregatedDataCollection implements Iterable<AggregatedData> {
private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>(); private final LinkedHashMap<Aggregate, AggregatedData> aggregatedDatas = new LinkedHashMap<>();
public void put(Aggregate aggregate, AggregatedData aggregatedData) { public void put(final Aggregate aggregate, final AggregatedData aggregatedData) {
aggregatedDatas.put(aggregate, aggregatedData); aggregatedDatas.put(aggregate, aggregatedData);
} }
@Override @Override
public Iterator<AggregatedData> iterator() { public Iterator<AggregatedData> iterator() {
return aggregatedDatas.values().iterator(); return aggregatedDatas.values().iterator();
} }
public Optional<AggregatedData> get(Aggregate aggregateType) { public Optional<AggregatedData> get(final Aggregate aggregateType) {
return Optional.ofNullable(aggregatedDatas.get(aggregateType)); return Optional.ofNullable(aggregatedDatas.get(aggregateType));
} }
} }

29
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatedDataEntry.java
View File

@@ -1,18 +1,21 @@
package org.lucares.pdb.plot.api; package org.lucares.pdb.plot.api;
public class AggregatedDataEntry { public class AggregatedDataEntry {
private final double epochSeconds; private final double epochSeconds;
private final long value; private final long value;
public AggregatedDataEntry(double epochSeconds, long value) {
super(); public AggregatedDataEntry(final double epochSeconds, final long value) {
this.epochSeconds = epochSeconds; super();
this.value = value; this.epochSeconds = epochSeconds;
} this.value = value;
public double getEpochSeconds() { }
return epochSeconds;
} public double getEpochSeconds() {
public long getValue() { return epochSeconds;
return value; }
}
public long getValue() {
return value;
}
} }

36
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java
View File

@@ -4,26 +4,26 @@ import java.io.IOException;
import java.util.List; import java.util.List;
public class AggregatorCollection { public class AggregatorCollection {
private final List<CustomAggregator> aggregators; private final List<CustomAggregator> aggregators;
public AggregatorCollection(List<CustomAggregator> aggregators) { public AggregatorCollection(final List<CustomAggregator> aggregators) {
this.aggregators = aggregators; this.aggregators = aggregators;
}
public void addValue(boolean valueIsInYRange, long epochMilli, long value) {
for (CustomAggregator aggregator : aggregators) {
aggregator.addValue(valueIsInYRange, epochMilli, value);
} }
}
public AggregatedDataCollection getAggregatedData() throws IOException { public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
for (final CustomAggregator aggregator : aggregators) {
AggregatedDataCollection result = new AggregatedDataCollection(); aggregator.addValue(valueIsInYRange, epochMilli, value);
}
for (CustomAggregator aggregator : aggregators) { }
result.put(aggregator.getType(), aggregator.getAggregatedData());
public AggregatedDataCollection getAggregatedData() throws IOException {
final AggregatedDataCollection result = new AggregatedDataCollection();
for (final CustomAggregator aggregator : aggregators) {
result.put(aggregator.getType(), aggregator.getAggregatedData());
}
return result;
} }
return result;
}
} }

18
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Appender.java
View File

@@ -3,15 +3,15 @@ package org.lucares.pdb.plot.api;
import java.util.Locale; import java.util.Locale;
public interface Appender { public interface Appender {
default void appendln(final StringBuilder builder, final String string) { default void appendln(final StringBuilder builder, final String string) {
builder.append(string + "\n"); builder.append(string + "\n");
} }
default void appendfln(final StringBuilder builder, final String format, final Object... args) { default void appendfln(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US,format + "\n", args)); builder.append(String.format(Locale.US, format + "\n", args));
} }
default void appendf(final StringBuilder builder, final String format, final Object... args) { default void appendf(final StringBuilder builder, final String format, final Object... args) {
builder.append(String.format(Locale.US,format, args)); builder.append(String.format(Locale.US, format, args));
} }
} }

2
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AxisScale.java
View File

@@ -1,5 +1,5 @@
package org.lucares.pdb.plot.api; package org.lucares.pdb.plot.api;
public enum AxisScale { public enum AxisScale {
LINEAR, LOG10 LINEAR, LOG10
} }

162
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java
View File

@@ -15,114 +15,114 @@ import org.lucares.collections.LongLongHashMap;
public class CumulativeDistributionCustomAggregator implements CustomAggregator { public class CumulativeDistributionCustomAggregator implements CustomAggregator {
private final static int POINTS = 500; private final static int POINTS = 500;
private static final class ToPercentiles implements LongLongConsumer { private static final class ToPercentiles implements LongLongConsumer {
private long cumulativeCount = 0; private long cumulativeCount = 0;
private long maxValue = 0; private long maxValue = 0;
private final LinkedHashMap<Double, Long> percentiles = new LinkedHashMap<>(POINTS); private final LinkedHashMap<Double, Long> percentiles = new LinkedHashMap<>(POINTS);
private final double stepSize; private final double stepSize;
private double lastPercentile; private double lastPercentile;
private double nextPercentile; private double nextPercentile;
private final long totalValues; private final long totalValues;
public ToPercentiles(final long totalValues) { public ToPercentiles(final long totalValues) {
this.totalValues = totalValues; this.totalValues = totalValues;
stepSize = 100.0 / POINTS; stepSize = 100.0 / POINTS;
nextPercentile = stepSize; nextPercentile = stepSize;
} }
@Override @Override
public void accept(final long duration, final long count) { public void accept(final long duration, final long count) {
maxValue = duration; maxValue = duration;
cumulativeCount += count; cumulativeCount += count;
final double newPercentile = cumulativeCount * 100.0 / totalValues; final double newPercentile = cumulativeCount * 100.0 / totalValues;
if (newPercentile >= nextPercentile) { if (newPercentile >= nextPercentile) {
double currentPercentile = lastPercentile + stepSize; double currentPercentile = lastPercentile + stepSize;
while (currentPercentile <= newPercentile) { while (currentPercentile <= newPercentile) {
percentiles.put(currentPercentile, duration); percentiles.put(currentPercentile, duration);
currentPercentile += stepSize; currentPercentile += stepSize;
} }
nextPercentile = currentPercentile; nextPercentile = currentPercentile;
lastPercentile = currentPercentile - stepSize; lastPercentile = currentPercentile - stepSize;
} }
} }
public long getMaxValue() { public long getMaxValue() {
return maxValue; return maxValue;
} }
public LinkedHashMap<Double, Long> getPercentiles() { public LinkedHashMap<Double, Long> getPercentiles() {
return percentiles; return percentiles;
} }
} }
// the rather large initial capacity should prevent too many grow&re-hash phases // the rather large initial capacity should prevent too many grow&re-hash phases
private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75); private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
private long totalValues = 0; private long totalValues = 0;
private final Path tmpDir; private final Path tmpDir;
public CumulativeDistributionCustomAggregator(final Path tmpDir) { public CumulativeDistributionCustomAggregator(final Path tmpDir) {
this.tmpDir = tmpDir; this.tmpDir = tmpDir;
} }
@Override @Override
public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) { public void addValue(boolean valueIsInYRange, final long epochMilli, final long value) {
map.compute(value, 0, l -> l + 1); map.compute(value, 0, l -> l + 1);
totalValues++; totalValues++;
} }
@Override @Override
public AggregatedData getAggregatedData() throws IOException { public AggregatedData getAggregatedData() throws IOException {
final char separator = ','; final char separator = ',';
final char newline = '\n'; final char newline = '\n';
final ToPercentiles toPercentiles = new ToPercentiles(totalValues); final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
map.forEachOrdered(toPercentiles); map.forEachOrdered(toPercentiles);
final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile()); final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
try (final Writer output = new BufferedWriter( try (final Writer output = new BufferedWriter(
new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) { new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
final StringBuilder data = new StringBuilder(); final StringBuilder data = new StringBuilder();
if (map.size() > 0) { if (map.size() > 0) {
// compute the percentiles // compute the percentiles
toPercentiles.getPercentiles().forEach((percentile, value) -> { toPercentiles.getPercentiles().forEach((percentile, value) -> {
data.append(percentile); data.append(percentile);
data.append(separator); data.append(separator);
data.append(value); data.append(value);
data.append(newline); data.append(newline);
}); });
final long maxValue = toPercentiles.getMaxValue(); final long maxValue = toPercentiles.getMaxValue();
data.append(100); data.append(100);
data.append(separator); data.append(separator);
data.append(maxValue); data.append(maxValue);
data.append(newline); data.append(newline);
} }
output.write(data.toString()); output.write(data.toString());
} }
final String title = String.format("cumulative distribution"); final String title = String.format("cumulative distribution");
return new AggregatedData(title, dataFile); return new AggregatedData(title, dataFile);
} }
@Override @Override
public Aggregate getType() { public Aggregate getType() {
return Aggregate.CUM_DISTRIBUTION; return Aggregate.CUM_DISTRIBUTION;
} }
} }

128
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionHandler.java
View File

@@ -14,74 +14,74 @@ import org.lucares.recommind.logs.AxisSettings.Type;
public class CumulativeDistributionHandler extends AggregateHandler { public class CumulativeDistributionHandler extends AggregateHandler {
@Override @Override
public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings, public CustomAggregator createCustomAggregator(final Path tmpDir, PlotSettings plotSettings,
final long fromEpochMilli, final long toEpochMilli) { final long fromEpochMilli, final long toEpochMilli) {
return new CumulativeDistributionCustomAggregator(tmpDir); return new CumulativeDistributionCustomAggregator(tmpDir);
}
public CumulativeDistributionHandler() {
}
@Override
Type getAxisType(GnuplotAxis axis) {
switch (axis) {
case X1:
case X2:
return Type.Percent;
case Y1:
case Y2:
return Type.Duration;
default:
throw new IllegalArgumentException("Unexpected value: " + axis);
} }
}
@Override public CumulativeDistributionHandler() {
public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = new AxisSettings();
result.setLabel("Cumulative Distribution");
result.setType(Type.Percent);
result.setAxis(getxAxis());
result.setFormat("%.0f%%");
result.setTicIncrement(computeTicIncrement(settings));
result.setFrom("0");
result.setTo("100");
return result;
}
private int computeTicIncrement(GnuplotSettings settings) {
int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
if (widthByFontSize < 50) {
return 20;
} else if (widthByFontSize < 75) {
return 10;
} else {
return 5;
} }
}
@Override @Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle, Type getAxisType(GnuplotAxis axis) {
Optional<String> title) { switch (axis) {
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", // case X1:
aggregatedData.getDataFile().getAbsolutePath(), // case X2:
gnuplotTitle(title), // return Type.Percent;
gnuplotXYAxis(), // case Y1:
lineStyle.darker()// case Y2:
); return Type.Duration;
} default:
throw new IllegalArgumentException("Unexpected value: " + axis);
}
}
@Override @Override
public Aggregate getAggregateType() { public AxisSettings createYAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
return Aggregate.CUM_DISTRIBUTION; AxisSettings result = AxisTime.createYAxis(settings, dataSeries);
} result.setAxis(getyAxis());
return result;
}
@Override
public AxisSettings createXAxisSettings(GnuplotSettings settings, Collection<DataSeries> dataSeries) {
AxisSettings result = new AxisSettings();
result.setLabel("Cumulative Distribution");
result.setType(Type.Percent);
result.setAxis(getxAxis());
result.setFormat("%.0f%%");
result.setTicIncrement(computeTicIncrement(settings));
result.setFrom("0");
result.setTo("100");
return result;
}
private int computeTicIncrement(GnuplotSettings settings) {
int widthByFontSize = settings.getWidth() / GnuplotSettings.TICKS_FONT_SIZE;
if (widthByFontSize < 50) {
return 20;
} else if (widthByFontSize < 75) {
return 10;
} else {
return 5;
}
}
@Override
public void addPlot(StringBuilder result, AggregatedData aggregatedData, LineStyle lineStyle,
Optional<String> title) {
appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
aggregatedData.getDataFile().getAbsolutePath(), //
gnuplotTitle(title), //
gnuplotXYAxis(), //
lineStyle.darker()//
);
}
@Override
public Aggregate getAggregateType() {
return Aggregate.CUM_DISTRIBUTION;
}
} }

Some files were not shown because too many files have changed in this diff Show More