andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

apply new code formatter and save action

Browse Source
This commit is contained in:
Andreas Huber
2019-11-24 10:20:43 +01:00
parent 5ea82c6a4c
commit 06b379494f
184 changed files with 13455 additions and 13489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -5,60 +5,60 @@ import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.internal.ParititionId;
public class Doc {
private final Tags tags;
private final Tags tags;
/**
* the block number used by {@link BSFile}
*/
private final long rootBlockNumber;
/**
* the block number used by {@link BSFile}
*/
private final long rootBlockNumber;
private ParititionId partitionId;
private ParititionId partitionId;
/**
* Initializes a new document.
* <p>
* The path can be {@code null}. If path is {@code null}, then
* {@code offsetInListingFile} must be set. The path will be initialized lazily
* when needed.
* <p>
* This is used to reduce the memory footprint.
*
* @param tags
* @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath}
*/
public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) {
this.partitionId = partitionId;
this.tags = tags;
this.rootBlockNumber = rootBlockNumber;
}
/**
* Initializes a new document.
* <p>
* The path can be {@code null}. If path is {@code null}, then
* {@code offsetInListingFile} must be set. The path will be initialized lazily
* when needed.
* <p>
* This is used to reduce the memory footprint.
*
* @param tags
* @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath}
*/
public Doc(final ParititionId partitionId, final Tags tags, final long rootBlockNumber) {
this.partitionId = partitionId;
this.tags = tags;
this.rootBlockNumber = rootBlockNumber;
}
public ParititionId getPartitionId() {
return partitionId;
}
public ParititionId getPartitionId() {
return partitionId;
}
public Tags getTags() {
return tags;
}
public Tags getTags() {
return tags;
}
/**
* the block number used by {@link BSFile}
*
* @return the root block number of this document
*/
public long getRootBlockNumber() {
return rootBlockNumber;
}
/**
* the block number used by {@link BSFile}
*
* @return the root block number of this document
*/
public long getRootBlockNumber() {
return rootBlockNumber;
}
public void setPartitionId(final ParititionId partitionId) {
this.partitionId = partitionId;
}
public void setPartitionId(final ParititionId partitionId) {
this.partitionId = partitionId;
}
@Override
public String toString() {
return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
@Override
public String toString() {
return "Doc [partitionId=" + partitionId + ", tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
}

8
data-store/src/main/java/org/lucares/pdb/datastore/InvalidValueException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class InvalidValueException extends IllegalArgumentException {
private static final long serialVersionUID = -8707541995666127297L;
private static final long serialVersionUID = -8707541995666127297L;
public InvalidValueException(final String msg) {
super(msg);
}
public InvalidValueException(final String msg) {
super(msg);
}
}

132
data-store/src/main/java/org/lucares/pdb/datastore/PdbFile.java
View File

@@ -14,86 +14,86 @@ import org.lucares.pdb.diskstorage.DiskStorage;
public class PdbFile {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private final PartitionDiskStore partitionDiskStorage;
private final PartitionDiskStore partitionDiskStorage;
public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) {
this.partitionDiskStorage = partitionDiskStorage;
}
public PdbFileToLongStream(final PartitionDiskStore partitionDiskStorage) {
this.partitionDiskStorage = partitionDiskStorage;
}
@Override
public Stream<LongList> apply(final PdbFile pdbFile) {
final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId());
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
return bsFile.streamOfLongLists();
}
}
@Override
public Stream<LongList> apply(final PdbFile pdbFile) {
final DiskStorage diskStorage = partitionDiskStorage.getExisting(pdbFile.getPartitionId());
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
return bsFile.streamOfLongLists();
}
}
private final Tags tags;
private final Tags tags;
/**
* The rootBlockNumber to be used by {@link BSFile}
*/
private final long rootBlockNumber;
/**
* The rootBlockNumber to be used by {@link BSFile}
*/
private final long rootBlockNumber;
private final ParititionId partitionId;
private final ParititionId partitionId;
public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) {
this.partitionId = partitionId;
this.rootBlockNumber = rootBlockNumber;
this.tags = tags;
}
public PdbFile(final ParititionId partitionId, final long rootBlockNumber, final Tags tags) {
this.partitionId = partitionId;
this.rootBlockNumber = rootBlockNumber;
this.tags = tags;
}
public Tags getTags() {
return tags;
}
public Tags getTags() {
return tags;
}
public long getRootBlockNumber() {
return rootBlockNumber;
}
public long getRootBlockNumber() {
return rootBlockNumber;
}
public ParititionId getPartitionId() {
return partitionId;
}
public ParititionId getPartitionId() {
return partitionId;
}
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) {
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final PartitionDiskStore diskStorage) {
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
return longStream;
}
return longStream;
}
@Override
public String toString() {
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId="+partitionId+"]";
}
@Override
public String toString() {
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + ", partitionId=" + partitionId + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
}

172
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -1,105 +1,105 @@
package org.lucares.pdb.datastore;
public class Proposal implements Comparable<Proposal> {
private final String proposedTag;
private final String proposedTag;
private final String proposedQuery;
private final String proposedQuery;
private final boolean hasResults;
private final boolean hasResults;
private final String newQuery;
private final String newQuery;
private final int newCaretPosition;
private final int newCaretPosition;
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults,
final String newQuery, final int newCaretPosition) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.hasResults = hasResults;
this.newQuery = newQuery;
this.newCaretPosition = newCaretPosition;
}
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults,
final String newQuery, final int newCaretPosition) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.hasResults = hasResults;
this.newQuery = newQuery;
this.newCaretPosition = newCaretPosition;
}
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.hasResults = hasResults;
this.newQuery = proposal.newQuery;
this.newCaretPosition = proposal.newCaretPosition;
}
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.hasResults = hasResults;
this.newQuery = proposal.newQuery;
this.newCaretPosition = proposal.newCaretPosition;
}
public String getProposedTag() {
return proposedTag;
}
public String getProposedTag() {
return proposedTag;
}
public String getProposedQuery() {
return proposedQuery;
}
public String getProposedQuery() {
return proposedQuery;
}
public boolean hasResults() {
return hasResults;
}
public boolean hasResults() {
return hasResults;
}
public String getNewQuery() {
return newQuery;
}
public String getNewQuery() {
return newQuery;
}
public int getNewCaretPosition() {
return newCaretPosition;
}
public int getNewCaretPosition() {
return newCaretPosition;
}
@Override
public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults="
+ hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]";
}
@Override
public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", hasResults="
+ hasResults + ", newQuery=" + newQuery + ", newCaretPosition=" + newCaretPosition + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + newCaretPosition;
result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode());
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + newCaretPosition;
result = prime * result + ((newQuery == null) ? 0 : newQuery.hashCode());
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (newCaretPosition != other.newCaretPosition)
return false;
if (newQuery == null) {
if (other.newQuery != null)
return false;
} else if (!newQuery.equals(other.newQuery))
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
} else if (!proposedQuery.equals(other.proposedQuery))
return false;
if (proposedTag == null) {
if (other.proposedTag != null)
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (newCaretPosition != other.newCaretPosition)
return false;
if (newQuery == null) {
if (other.newQuery != null)
return false;
} else if (!newQuery.equals(other.newQuery))
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
} else if (!proposedQuery.equals(other.proposedQuery))
return false;
if (proposedTag == null) {
if (other.proposedTag != null)
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
return true;
}
@Override
public int compareTo(final Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
@Override
public int compareTo(final Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
}

8
data-store/src/main/java/org/lucares/pdb/datastore/ReadException.java
View File

@@ -2,9 +2,9 @@ package org.lucares.pdb.datastore;
public class ReadException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public ReadException(final RuntimeException e) {
super(e);
}
public ReadException(final RuntimeException e) {
super(e);
}
}

20
data-store/src/main/java/org/lucares/pdb/datastore/ReadRuntimeException.java
View File

@@ -2,17 +2,17 @@ package org.lucares.pdb.datastore;
public class ReadRuntimeException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause);
}
public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause);
}
public ReadRuntimeException(final String message) {
super(message);
}
public ReadRuntimeException(final String message) {
super(message);
}
public ReadRuntimeException(final Throwable cause) {
super(cause);
}
public ReadRuntimeException(final Throwable cause) {
super(cause);
}
}

14
data-store/src/main/java/org/lucares/pdb/datastore/WriteException.java
View File

@@ -2,14 +2,14 @@ package org.lucares.pdb.datastore;
public class WriteException extends RuntimeException {
private static final long serialVersionUID = 1L;
private static final long serialVersionUID = 1L;
public WriteException(final String message, final Throwable cause) {
super(message, cause);
}
public WriteException(final String message, final Throwable cause) {
super(message, cause);
}
public WriteException(final Throwable cause) {
super(cause);
}
public WriteException(final Throwable cause) {
super(cause);
}
}

670
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -39,381 +39,381 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore implements AutoCloseable {
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
public static final char LISTING_FILE_SEPARATOR = ',';
public static final char LISTING_FILE_SEPARATOR = ',';
public static final String SUBDIR_STORAGE = "storage";
public static final String SUBDIR_STORAGE = "storage";
// used to generate doc ids that are
// a) unique
// b) monotonically increasing (this is, so that we don't have to sort the doc
// ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
// used to generate doc ids that are
// a) unique
// b) monotonically increasing (this is, so that we don't have to sort the doc
// ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
public static Tag TAG_ALL_DOCS = null;
public static Tag TAG_ALL_DOCS = null;
private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc;
private final PartitionPersistentMap<Long, Doc, Doc> docIdToDoc;
private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId;
private final PartitionPersistentMap<Tags, Long, Long> tagsToDocId;
private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId;
private final PartitionPersistentMap<Tag, Long, Long> tagToDocsId;
private final QueryCompletionIndex queryCompletionIndex;
private final QueryCompletionIndex queryCompletionIndex;
// A Doc will never be changed once it is created. Therefore we can cache them
// easily.
private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000);
// A Doc will never be changed once it is created. Therefore we can cache them
// easily.
private final HotEntryCache<Long, Doc> docIdToDocCache = new HotEntryCache<>(Duration.ofMinutes(30), 100_000);
private final HotEntryCache<Tags, PdbWriter> writerCache;
private final HotEntryCache<Tags, PdbWriter> writerCache;
private final PartitionDiskStore diskStorage;
private final Path storageBasePath;
private final PartitionDiskStore diskStorage;
private final Path storageBasePath;
public DataStore(final Path dataDirectory) throws IOException {
storageBasePath = storageDirectory(dataDirectory);
public DataStore(final Path dataDirectory) throws IOException {
storageBasePath = storageDirectory(dataDirectory);
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it
// must be initialized after the string compressor has been created
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = new Tag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the StringCompressor internally, so it
// must be initialized after the string compressor has been created
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs",
new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagToDocsId = new PartitionPersistentMap<>(storageBasePath, "keyToValueToDocIdsIndex.bs",
new TagEncoderDecoder(), PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(),
PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
tagsToDocId = new PartitionPersistentMap<>(storageBasePath, "tagsToDocIdIndex.bs", new TagsEncoderDecoder(),
PartitionAwareWrapper.wrap(PersistentMap.LONG_CODER));
docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER,
new DocEncoderDecoder());
docIdToDoc = new PartitionPersistentMap<>(storageBasePath, "docIdToDocIndex.bs", PersistentMap.LONG_CODER,
new DocEncoderDecoder());
queryCompletionIndex = new QueryCompletionIndex(storageBasePath);
queryCompletionIndex = new QueryCompletionIndex(storageBasePath);
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000);
writerCache.addListener((key, value) -> value.close());
}
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), 1000);
writerCache.addListener((key, value) -> value.close());
}
private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv");
}
private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv");
}
public static Path storageDirectory(final Path dataDirectory) throws IOException {
return dataDirectory.resolve(SUBDIR_STORAGE);
}
public static Path storageDirectory(final Path dataDirectory) throws IOException {
return dataDirectory.resolve(SUBDIR_STORAGE);
}
public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags);
writer.write(dateAsEpochMilli, value);
}
public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags);
writer.write(dateAsEpochMilli, value);
}
// visible for test
QueryCompletionIndex getQueryCompletionIndex() {
return queryCompletionIndex;
}
// visible for test
QueryCompletionIndex getQueryCompletionIndex() {
return queryCompletionIndex;
}
public long createNewFile(final ParititionId partitionId, final Tags tags) {
try {
final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
public long createNewFile(final ParititionId partitionId, final Tags tags) {
try {
final long newFilesRootBlockOffset = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
final long docId = createUniqueDocId();
final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset);
docIdToDoc.putValue(partitionId, docId, doc);
final long docId = createUniqueDocId();
final Doc doc = new Doc(partitionId, tags, newFilesRootBlockOffset);
docIdToDoc.putValue(partitionId, docId, doc);
final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId);
Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags);
final Long oldDocId = tagsToDocId.putValue(partitionId, tags, docId);
Preconditions.checkNull(oldDocId, "There must be at most one document for tags: {0}", tags);
// store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS);
for (final Tag tag : ts) {
// store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS);
for (final Tag tag : ts) {
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
if (diskStoreOffsetForDocIdsOfTag == null) {
diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag);
}
if (diskStoreOffsetForDocIdsOfTag == null) {
diskStoreOffsetForDocIdsOfTag = diskStorage.allocateBlock(partitionId, BSFile.BLOCK_SIZE);
tagToDocsId.putValue(partitionId, tag, diskStoreOffsetForDocIdsOfTag);
}
try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag,
partitionId)) {
docIdsOfTag.append(docId);
}
}
try (final LongStreamFile docIdsOfTag = diskStorage.streamExistingFile(diskStoreOffsetForDocIdsOfTag,
partitionId)) {
docIdsOfTag.append(docId);
}
}
// index the tags, so that we can efficiently find all possible values for a
// field in a query
queryCompletionIndex.addTags(partitionId, tags);
// index the tags, so that we can efficiently find all possible values for a
// field in a query
queryCompletionIndex.addTags(partitionId, tags);
return newFilesRootBlockOffset;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
return newFilesRootBlockOffset;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private long createUniqueDocId() {
return NEXT_DOC_ID.getAndIncrement();
}
private long createUniqueDocId() {
return NEXT_DOC_ID.getAndIncrement();
}
public List<PdbFile> getFilesForQuery(final Query query) {
public List<PdbFile> getFilesForQuery(final Query query) {
final List<Doc> searchResult = search(query);
if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results.");
}
final List<Doc> searchResult = search(query);
if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results.");
}
final List<PdbFile> result = toPdbFiles(searchResult);
return result;
}
final List<PdbFile> result = toPdbFiles(searchResult);
return result;
}
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) {
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) {
final ParititionId partitionId = document.getPartitionId();
final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags);
final ParititionId partitionId = document.getPartitionId();
final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(partitionId, rootBlockNumber, tags);
result.add(pdbFile);
}
return result;
}
result.add(pdbFile);
}
return result;
}
public List<Doc> search(final Query query) {
try {
final List<Doc> result = new ArrayList<>();
public List<Doc> search(final Query query) {
try {
final List<Doc> result = new ArrayList<>();
final PartitionLongList docIdsList = executeQuery(query);
LOGGER.trace("query {} found {} docs", query, docIdsList.size());
final List<Doc> docs = mapDocIdsToDocs(docIdsList);
result.addAll(docs);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public int count(final Query query) {
final PartitionLongList docIdsList = executeQuery(query);
return docIdsList.size();
}
public List<String> getAvailableFields(final DateTimeRange dateRange) {
final Set<String> keys = new HashSet<>();
final Tag keyPrefix = new Tag("", ""); // will find everything
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString()));
keys.remove(ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys);
Collections.sort(result);
return result;
}
private PartitionLongList executeQuery(final Query query) {
final long start = System.nanoTime();
synchronized (docIdToDoc) {
final Expression expression = QueryLanguageParser.parse(query.getQuery());
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId,
diskStorage);
final PartitionLongList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query,
(System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}
}
private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException {
final List<Doc> result = new ArrayList<>(docIdsList.size());
synchronized (docIdToDoc) {
final long start = System.nanoTime();
for (final ParititionId partitionId : docIdsList) {
final LongList docIds = docIdsList.get(partitionId);
for (int i = 0; i < docIds.size(); i++) {
final long docId = docIds.get(i);
final Doc doc = getDocByDocId(partitionId, docId);
Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist.");
result.add(doc);
}
}
MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(),
(System.nanoTime() - start) / 1_000_000.0);
}
return result;
}
public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) {
final Long docId = tagsToDocId.getValue(partitionId, tags);
if (docId != null) {
final Doc doc = getDocByDocId(partitionId, docId);
return Optional.of(doc);
}
return Optional.empty();
}
public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) {
final List<Doc> result = new ArrayList<>();
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags);
for (final Long docId : docIds) {
if (docId != null) {
final Doc doc = getDocByDocId(dateRange, docId);
result.add(doc);
}
}
return result;
}
private Doc getDocByDocId(final ParititionId partitionId, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
return docIdToDoc.getValue(partitionId, documentId);
});
}
private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId);
if (docIds.size() == 1) {
return docIds.get(0);
} else if (docIds.size() > 1) {
throw new IllegalStateException(
"Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds);
}
throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId);
});
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex);
final List<Proposal> proposals = newProposerParser.propose(query);
LOGGER.debug("Proposals for query {}: {}", query, proposals);
return proposals;
}
public PartitionDiskStore getDiskStorage() {
return diskStorage;
}
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags));
}
// visible for test
long sizeWriterCache() {
return writerCache.size();
}
private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) {
final Optional<Doc> docsForTags = getByTags(partitionId, tags);
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
} catch (final RuntimeException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(partitionId, tags);
}
return writer;
}
private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) {
final long rootBlockNumber = createNewFile(partitionId, tags);
final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags);
return result;
}
@Override
public void close() throws RuntimeIOException {
try {
// we cannot simply clear the cache, because the cache implementation (Guava at
// the time of writing) handles eviction events asynchronously.
forEachWriter(cachedWriter -> {
try {
cachedWriter.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
} finally {
try {
diskStorage.close();
} finally {
tagToDocsId.close();
}
}
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> {
try {
consumer.accept(writer);
} catch (final RuntimeException e) {
LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
final PartitionLongList docIdsList = executeQuery(query);
LOGGER.trace("query {} found {} docs", query, docIdsList.size());
final List<Doc> docs = mapDocIdsToDocs(docIdsList);
result.addAll(docs);
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public int count(final Query query) {
final PartitionLongList docIdsList = executeQuery(query);
return docIdsList.size();
}
public List<String> getAvailableFields(final DateTimeRange dateRange) {
final Set<String> keys = new HashSet<>();
final Tag keyPrefix = new Tag("", ""); // will find everything
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tags, __) -> keys.add(tags.getKeyAsString()));
keys.remove(ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys);
Collections.sort(result);
return result;
}
private PartitionLongList executeQuery(final Query query) {
final long start = System.nanoTime();
synchronized (docIdToDoc) {
final Expression expression = QueryLanguageParser.parse(query.getQuery());
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(query.getDateRange(), tagToDocsId,
diskStorage);
final PartitionLongList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results ", query,
(System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}
}
private List<Doc> mapDocIdsToDocs(final PartitionLongList docIdsList) throws IOException {
final List<Doc> result = new ArrayList<>(docIdsList.size());
synchronized (docIdToDoc) {
final long start = System.nanoTime();
for (final ParititionId partitionId : docIdsList) {
final LongList docIds = docIdsList.get(partitionId);
for (int i = 0; i < docIds.size(); i++) {
final long docId = docIds.get(i);
final Doc doc = getDocByDocId(partitionId, docId);
Objects.requireNonNull(doc, "Doc with id " + docId + " did not exist.");
result.add(doc);
}
}
MAP_DOCS_TO_DOCID.debug("mapDocIdsToDocs({}): {}ms", docIdsList.size(),
(System.nanoTime() - start) / 1_000_000.0);
}
return result;
}
public Optional<Doc> getByTags(final ParititionId partitionId, final Tags tags) {
final Long docId = tagsToDocId.getValue(partitionId, tags);
if (docId != null) {
final Doc doc = getDocByDocId(partitionId, docId);
return Optional.of(doc);
}
return Optional.empty();
}
public List<Doc> getByTags(final DateTimeRange dateRange, final Tags tags) {
final List<Doc> result = new ArrayList<>();
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Long> docIds = tagsToDocId.getValues(datePartitioner, tags);
for (final Long docId : docIds) {
if (docId != null) {
final Doc doc = getDocByDocId(dateRange, docId);
result.add(doc);
}
}
return result;
}
private Doc getDocByDocId(final ParititionId partitionId, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
return docIdToDoc.getValue(partitionId, documentId);
});
}
private Doc getDocByDocId(final DateTimeRange dateRange, final Long docId) {
return docIdToDocCache.putIfAbsent(docId, documentId -> {
final DatePartitioner datePartitioner = new DatePartitioner(dateRange);
final List<Doc> docIds = docIdToDoc.getValues(datePartitioner, documentId);
if (docIds.size() == 1) {
return docIds.get(0);
} else if (docIds.size() > 1) {
throw new IllegalStateException(
"Found multiple documents for " + dateRange + " and docId " + documentId + ": " + docIds);
}
throw new IllegalStateException("Found no documents for " + dateRange + " and docId " + documentId);
});
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex);
final List<Proposal> proposals = newProposerParser.propose(query);
LOGGER.debug("Proposals for query {}: {}", query, proposals);
return proposals;
}
public PartitionDiskStore getDiskStorage() {
return diskStorage;
}
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
return writerCache.putIfAbsent(tags, t -> getWriterInternal(partitionId, tags));
}
// visible for test
long sizeWriterCache() {
return writerCache.size();
}
private PdbWriter getWriterInternal(final ParititionId partitionId, final Tags tags) {
final Optional<Doc> docsForTags = getByTags(partitionId, tags);
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
} catch (final RuntimeException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(partitionId, tags);
}
return writer;
}
private PdbWriter newPdbWriter(final ParititionId partitionId, final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final ParititionId partitionId, final Tags tags) {
final long rootBlockNumber = createNewFile(partitionId, tags);
final PdbFile result = new PdbFile(partitionId, rootBlockNumber, tags);
return result;
}
@Override
public void close() throws RuntimeIOException {
try {
// we cannot simply clear the cache, because the cache implementation (Guava at
// the time of writing) handles eviction events asynchronously.
forEachWriter(cachedWriter -> {
try {
cachedWriter.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
} finally {
try {
diskStorage.close();
} finally {
tagToDocsId.close();
}
}
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> {
try {
consumer.accept(writer);
} catch (final RuntimeException e) {
LOGGER.warn("Exception while applying consumer to PdbWriter for " + writer.getPdbFile(), e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
}

268
data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java
View File

@@ -19,178 +19,178 @@ import org.lucares.pdb.api.DateTimeRange;
public class DateIndexExtension {
/**
* This date pattern defines the resolution of the date index
*/
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
/**
* This date pattern defines the resolution of the date index
*/
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
// visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
// visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null);
private static final AtomicReference<DatePrefixAndRange> LAST_ACCESSED = new AtomicReference<>(null);
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>();
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>();
OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) {
OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) {
result.add(toDateIndexPrefix(current));
current = current.plusMonths(1);
result.add(toDateIndexPrefix(current));
current = current.plusMonths(1);
}
result.add(toDateIndexPrefix(dateRange.getEnd()));
}
result.add(toDateIndexPrefix(dateRange.getEnd()));
return result;
}
return result;
}
static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN);
}
static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN);
}
public static ParititionId toPartitionId(final long epochMilli) {
String result;
final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get();
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
&& lastAccessed.getMaxEpochMilli() >= epochMilli) {
result = lastAccessed.getDatePrefix();
} else {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
public static ParititionId toPartitionId(final long epochMilli) {
String result;
final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get();
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
&& lastAccessed.getMaxEpochMilli() >= epochMilli) {
result = lastAccessed.getDatePrefix();
} else {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
LAST_ACCESSED.set(newValue);
} else {
result = value.getValue().getDatePrefix();
LAST_ACCESSED.set(value.getValue());
}
}
return new ParititionId(result);
}
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
LAST_ACCESSED.set(newValue);
} else {
result = value.getValue().getDatePrefix();
LAST_ACCESSED.set(value.getValue());
}
}
return new ParititionId(result);
}
public static String toDateIndexPrefix(final long epochMilli) {
public static String toDateIndexPrefix(final long epochMilli) {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
String result;
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
} else {
result = value.getValue().getDatePrefix();
}
String result;
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
} else {
result = value.getValue().getDatePrefix();
}
return result;
}
return result;
}
/**
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final
* Collection<? extends PartitionId> availablePartitionIds) instead
*
* @param dateRange
* @return
*/
static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) {
final List<ParititionId> result = new ArrayList<>();
/**
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final
* Collection<? extends PartitionId> availablePartitionIds) instead
*
* @param dateRange
* @return
*/
static List<ParititionId> toPartitionIds(final DateTimeRange dateRange) {
final List<ParititionId> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
while (!current.isAfter(end)) {
final String id = current.format(DATE_PATTERN);
final ParititionId partitionId = new ParititionId(id);
result.add(partitionId);
current = current.plusMonths(1);
}
while (!current.isAfter(end)) {
final String id = current.format(DATE_PATTERN);
final ParititionId partitionId = new ParititionId(id);
result.add(partitionId);
current = current.plusMonths(1);
}
return result;
}
return result;
}
public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange,
final Collection<? extends ParititionId> availablePartitionIds) {
final Set<ParititionId> result = new LinkedHashSet<>();
public static Set<ParititionId> toPartitionIds(final DateTimeRange dateRange,
final Collection<? extends ParititionId> availablePartitionIds) {
final Set<ParititionId> result = new LinkedHashSet<>();
final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli());
final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli());
final ParititionId start = toPartitionId(dateRange.getStart().toInstant().toEpochMilli());
final ParititionId end = toPartitionId(dateRange.getEnd().toInstant().toEpochMilli());
for (final ParititionId partitionId : availablePartitionIds) {
if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) {
result.add(partitionId);
}
}
for (final ParititionId partitionId : availablePartitionIds) {
if (start.compareTo(partitionId) <= 0 && end.compareTo(partitionId) >= 0) {
result.add(partitionId);
}
}
return result;
}
return result;
}
public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
}
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
}
public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) {
final List<Long> result = new ArrayList<>();
public static List<Long> toDateIndexEpochMillis(final DateTimeRange dateRange) {
final List<Long> result = new ArrayList<>();
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
OffsetDateTime current = dateRange.getStart();
final OffsetDateTime end = dateRange.getEnd();
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
.withSecond(0).withNano(0);
while (!current.isAfter(end)) {
result.add(current.toInstant().toEpochMilli());
current = current.plusMonths(1);
}
while (!current.isAfter(end)) {
result.add(current.toInstant().toEpochMilli());
current = current.plusMonths(1);
}
return result;
}
return result;
}
public static ParititionId now() {
return toPartitionId(System.currentTimeMillis());
}
public static ParititionId now() {
return toPartitionId(System.currentTimeMillis());
}
}
class DatePrefixAndRange {
private final String datePrefix;
private final long minEpochMilli;
private final long maxEpochMilli;
private final String datePrefix;
private final long minEpochMilli;
private final long maxEpochMilli;
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super();
this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli;
}
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super();
this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli;
}
public String getDatePrefix() {
return datePrefix;
}
public String getDatePrefix() {
return datePrefix;
}
public long getMinEpochMilli() {
return minEpochMilli;
}
public long getMinEpochMilli() {
return minEpochMilli;
}
public long getMaxEpochMilli() {
return maxEpochMilli;
}
public long getMaxEpochMilli() {
return maxEpochMilli;
}
public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
}
public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
}
@Override
public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
}
@Override
public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
}
}

16
data-store/src/main/java/org/lucares/pdb/datastore/internal/DatePartitioner.java
View File

@@ -6,14 +6,14 @@ import org.lucares.pdb.api.DateTimeRange;
public class DatePartitioner implements PartitionIdSource {
private final DateTimeRange dateRange;
private final DateTimeRange dateRange;
public DatePartitioner(final DateTimeRange dateRange) {
this.dateRange = dateRange;
}
public DatePartitioner(final DateTimeRange dateRange) {
this.dateRange = dateRange;
}
@Override
public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) {
return DateIndexExtension.toPartitionIds(dateRange, availablePartitions);
}
@Override
public Set<ParititionId> toPartitionIds(final Set<? extends ParititionId> availablePartitions) {
return DateIndexExtension.toPartitionIds(dateRange, availablePartitions);
}
}

62
data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java
View File

@@ -8,43 +8,43 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
@Override
public byte[] encode(final Doc doc) {
@Override
public byte[] encode(final Doc doc) {
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().toBytes();
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().toBytes();
final byte[] result = new byte[rootBlockNumber.length + tags.length];
final byte[] result = new byte[rootBlockNumber.length + tags.length];
System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length);
System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length);
System.arraycopy(rootBlockNumber, 0, result, 0, rootBlockNumber.length);
System.arraycopy(tags, 0, result, rootBlockNumber.length, tags.length);
return result;
}
return result;
}
@Override
public Doc decode(final byte[] bytes) {
@Override
public Doc decode(final byte[] bytes) {
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(null, tags, rootBlockNumber);
}
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(null, tags, rootBlockNumber);
}
@Override
public Doc encodeValue(final Doc v) {
return v;
}
@Override
public Doc encodeValue(final Doc v) {
return v;
}
@Override
public Doc decodeValue(final ParititionId partitionId, final Doc t) {
if (t != null) {
t.setPartitionId(partitionId);
}
return t;
}
public byte[] getEmptyValue() {
return new byte[] {0};
}
@Override
public Doc decodeValue(final ParititionId partitionId, final Doc t) {
if (t != null) {
t.setPartitionId(partitionId);
}
return t;
}
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}

22
data-store/src/main/java/org/lucares/pdb/datastore/internal/GlobMatcher.java
View File

@@ -7,18 +7,18 @@ import org.lucares.pdb.datastore.lang.GloblikePattern;
public class GlobMatcher {
private final Pattern pattern;
private final Pattern pattern;
public GlobMatcher(final String globlike) {
pattern = GloblikePattern.globlikeToRegex(globlike);
}
public GlobMatcher(final String globlike) {
pattern = GloblikePattern.globlikeToRegex(globlike);
}
public GlobMatcher(final Iterable<String> globlikes) {
pattern = GloblikePattern.globlikeToRegex(globlikes);
}
public GlobMatcher(final Iterable<String> globlikes) {
pattern = GloblikePattern.globlikeToRegex(globlikes);
}
public boolean matches(final String s) {
final Matcher matcher = pattern.matcher(s);
return matcher.find();
}
public boolean matches(final String s) {
final Matcher matcher = pattern.matcher(s);
return matcher.find();
}
}

106
data-store/src/main/java/org/lucares/pdb/datastore/internal/ParititionId.java
View File

@@ -1,65 +1,65 @@
package org.lucares.pdb.datastore.internal;
public class ParititionId implements Comparable<ParititionId> {
private final String partitionId;
private final String partitionId;
/**
* Create a new partition id.
*
* @param partitionId the id, e.g. a time like 201902 (partition for entries of
* February 2019)
*/
public ParititionId(final String partitionId) {
super();
this.partitionId = partitionId;
}
/**
* Create a new partition id.
*
* @param partitionId the id, e.g. a time like 201902 (partition for entries of
* February 2019)
*/
public ParititionId(final String partitionId) {
super();
this.partitionId = partitionId;
}
public static ParititionId of(final String partitionId) {
return new ParititionId(partitionId);
}
public static ParititionId of(final String partitionId) {
return new ParititionId(partitionId);
}
@Override
public int compareTo(final ParititionId other) {
return partitionId.compareTo(other.getPartitionId());
}
@Override
public int compareTo(final ParititionId other) {
return partitionId.compareTo(other.getPartitionId());
}
/**
* @return the id, e.g. a time like 201902 (partition for entries of February
* 2019)
*/
public String getPartitionId() {
return partitionId;
}
/**
* @return the id, e.g. a time like 201902 (partition for entries of February
* 2019)
*/
public String getPartitionId() {
return partitionId;
}
@Override
public String toString() {
return partitionId;
}
@Override
public String toString() {
return partitionId;
}
/*
* non-standard hashcode implementation! This class is just a wrapper for
* string, so we delegate directly to String.hashCode().
*/
@Override
public int hashCode() {
return partitionId.hashCode();
}
/*
* non-standard hashcode implementation! This class is just a wrapper for
* string, so we delegate directly to String.hashCode().
*/
@Override
public int hashCode() {
return partitionId.hashCode();
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ParititionId other = (ParititionId) obj;
if (partitionId == null) {
if (other.partitionId != null)
return false;
} else if (!partitionId.equals(other.partitionId))
return false;
return true;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final ParititionId other = (ParititionId) obj;
if (partitionId == null) {
if (other.partitionId != null)
return false;
} else if (!partitionId.equals(other.partitionId))
return false;
return true;
}
}

4
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareEncoderDecoder.java
View File

@@ -4,7 +4,7 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public interface PartitionAwareEncoderDecoder<V, P> extends EncoderDecoder<P> {
public P encodeValue(V v);
public P encodeValue(V v);
public V decodeValue(ParititionId partitionId, P p);
public V decodeValue(ParititionId partitionId, P p);
}

54
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionAwareWrapper.java
View File

@@ -4,37 +4,37 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
public final class PartitionAwareWrapper<O> implements PartitionAwareEncoderDecoder<O, O> {
private final EncoderDecoder<O> delegate;
private final EncoderDecoder<O> delegate;
public PartitionAwareWrapper(final EncoderDecoder<O> delegate) {
this.delegate = delegate;
}
public PartitionAwareWrapper(final EncoderDecoder<O> delegate) {
this.delegate = delegate;
}
@Override
public byte[] encode(final O object) {
return delegate.encode(object);
}
@Override
public byte[] encode(final O object) {
return delegate.encode(object);
}
@Override
public O decode(final byte[] bytes) {
return delegate.decode(bytes);
}
@Override
public O decode(final byte[] bytes) {
return delegate.decode(bytes);
}
@Override
public O encodeValue(final O v) {
return v;
}
@Override
public O encodeValue(final O v) {
return v;
}
@Override
public O decodeValue(final ParititionId partitionId, final O p) {
return p;
}
@Override
public O decodeValue(final ParititionId partitionId, final O p) {
return p;
}
public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) {
return new PartitionAwareWrapper<>(encoder);
}
public byte[] getEmptyValue() {
return delegate.getEmptyValue();
}
public static <O> PartitionAwareEncoderDecoder<O, O> wrap(final EncoderDecoder<O> encoder) {
return new PartitionAwareWrapper<>(encoder);
}
public byte[] getEmptyValue() {
return delegate.getEmptyValue();
}
}

108
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionDiskStore.java
View File

@@ -14,68 +14,68 @@ import org.lucares.pdb.blockstorage.LongStreamFile;
import org.lucares.pdb.diskstorage.DiskStorage;
public class PartitionDiskStore {
private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>();
private final ConcurrentHashMap<ParititionId, DiskStorage> diskStorages = new ConcurrentHashMap<>();
private final Function<ParititionId, DiskStorage> creator;
private final Function<ParititionId, DiskStorage> supplier;
private final Function<ParititionId, DiskStorage> creator;
private final Function<ParititionId, DiskStorage> supplier;
public PartitionDiskStore(final Path storageBasePath, final String filename) {
public PartitionDiskStore(final Path storageBasePath, final String filename) {
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
final boolean isNew = !Files.exists(file);
final DiskStorage diskStorage = new DiskStorage(file, storageBasePath);
if (isNew) {
diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE);
}
return diskStorage;
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new DiskStorage(file, storageBasePath);
}
return null;
};
}
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
final boolean isNew = !Files.exists(file);
final DiskStorage diskStorage = new DiskStorage(file, storageBasePath);
if (isNew) {
diskStorage.ensureAlignmentForNewBlocks(BSFile.BLOCK_SIZE);
}
return diskStorage;
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new DiskStorage(file, storageBasePath);
}
return null;
};
}
public DiskStorage getExisting(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, supplier);
}
public DiskStorage getExisting(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, supplier);
}
public DiskStorage getCreateIfNotExists(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, creator);
}
public DiskStorage getCreateIfNotExists(final ParititionId partitionId) {
return diskStorages.computeIfAbsent(partitionId, creator);
}
public long allocateBlock(final ParititionId partitionId, final int blockSize) {
final DiskStorage diskStorage = getCreateIfNotExists(partitionId);
return diskStorage.allocateBlock(blockSize);
}
public long allocateBlock(final ParititionId partitionId, final int blockSize) {
final DiskStorage diskStorage = getCreateIfNotExists(partitionId);
return diskStorage.allocateBlock(blockSize);
}
public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) {
try {
final DiskStorage diskStorage = getExisting(partitionId);
return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public LongStreamFile streamExistingFile(final Long diskStoreOffsetForDocIdsOfTag, final ParititionId partitionId) {
try {
final DiskStorage diskStorage = getExisting(partitionId);
return LongStreamFile.existingFile(diskStoreOffsetForDocIdsOfTag, diskStorage);
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
public void close() {
final List<Throwable> throwables = new ArrayList<>();
public void close() {
final List<Throwable> throwables = new ArrayList<>();
for (final DiskStorage diskStorage : diskStorages.values()) {
try {
diskStorage.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
for (final DiskStorage diskStorage : diskStorages.values()) {
try {
diskStorage.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
}
}

2
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionIdSource.java
View File

@@ -3,5 +3,5 @@ package org.lucares.pdb.datastore.internal;
import java.util.Set;
public interface PartitionIdSource {
Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions);
Set<ParititionId> toPartitionIds(Set<? extends ParititionId> availablePartitions);
}

140
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionLongList.java
View File

@@ -9,87 +9,87 @@ import java.util.Set;
import org.lucares.collections.LongList;
public class PartitionLongList implements Iterable<ParititionId> {
private final Map<ParititionId, LongList> lists = new HashMap<>();
private final Map<ParititionId, LongList> lists = new HashMap<>();
public LongList put(final ParititionId partitionId, final LongList longList) {
return lists.put(partitionId, longList);
}
public LongList put(final ParititionId partitionId, final LongList longList) {
return lists.put(partitionId, longList);
}
public LongList get(final ParititionId partitionId) {
return lists.get(partitionId);
}
public LongList get(final ParititionId partitionId) {
return lists.get(partitionId);
}
@Override
public Iterator<ParititionId> iterator() {
return lists.keySet().iterator();
}
@Override
public Iterator<ParititionId> iterator() {
return lists.keySet().iterator();
}
public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
public static PartitionLongList intersection(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
if (x != null && y != null) {
final LongList intersection = LongList.intersection(x, y);
result.put(partitionId, intersection);
} else {
// one list is empty => the intersection is empty
}
}
return result;
}
if (x != null && y != null) {
final LongList intersection = LongList.intersection(x, y);
result.put(partitionId, intersection);
} else {
// one list is empty => the intersection is empty
}
}
return result;
}
public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
public static PartitionLongList union(final PartitionLongList a, final PartitionLongList b) {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> partitionIds = new HashSet<>();
partitionIds.addAll(a.lists.keySet());
partitionIds.addAll(b.lists.keySet());
for (final ParititionId partitionId : partitionIds) {
final LongList x = a.get(partitionId);
final LongList y = b.get(partitionId);
if (x != null && y != null) {
final LongList intersection = LongList.union(x, y);
result.put(partitionId, intersection);
} else if (x != null) {
result.put(partitionId, x.clone());
} else if (y != null) {
result.put(partitionId, y.clone());
}
}
return result;
}
if (x != null && y != null) {
final LongList intersection = LongList.union(x, y);
result.put(partitionId, intersection);
} else if (x != null) {
result.put(partitionId, x.clone());
} else if (y != null) {
result.put(partitionId, y.clone());
}
}
return result;
}
public int size() {
int size = 0;
public int size() {
int size = 0;
for (final LongList longList : lists.values()) {
size += longList.size();
}
for (final LongList longList : lists.values()) {
size += longList.size();
}
return size;
}
return size;
}
public boolean isSorted() {
for (final LongList longList : lists.values()) {
if (!longList.isSorted()) {
return false;
}
}
return true;
}
public boolean isSorted() {
for (final LongList longList : lists.values()) {
if (!longList.isSorted()) {
return false;
}
}
return true;
}
public void removeAll(final PartitionLongList remove) {
for (final ParititionId partitionId : lists.keySet()) {
final LongList removeLongList = remove.get(partitionId);
if (removeLongList != null) {
lists.get(partitionId).removeAll(removeLongList);
}
}
}
public void removeAll(final PartitionLongList remove) {
for (final ParititionId partitionId : lists.keySet()) {
final LongList removeLongList = remove.get(partitionId);
if (removeLongList != null) {
lists.get(partitionId).removeAll(removeLongList);
}
}
}
}

212
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java
View File

@@ -25,130 +25,130 @@ import org.lucares.pdb.map.Visitor;
*/
public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final Function<ParititionId, PersistentMap<K, P>> creator;
private final Function<ParititionId, PersistentMap<K, P>> supplier;
private final Function<ParititionId, PersistentMap<K, P>> creator;
private final Function<ParititionId, PersistentMap<K, P>> supplier;
private final PartitionAwareEncoderDecoder<V, P> valueEncoder;
private final PartitionAwareEncoderDecoder<V, P> valueEncoder;
public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder,
final PartitionAwareEncoderDecoder<V, P> valueEncoder) {
public PartitionPersistentMap(final Path storageBasePath, final String filename, final EncoderDecoder<K> keyEncoder,
final PartitionAwareEncoderDecoder<V, P> valueEncoder) {
this.valueEncoder = valueEncoder;
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
}
return null;
};
preload(storageBasePath);
}
this.valueEncoder = valueEncoder;
creator = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
};
supplier = partitionId -> {
final Path file = storageBasePath.resolve(partitionId.getPartitionId()).resolve(filename);
if (Files.exists(file)) {
return new PersistentMap<>(file, storageBasePath, keyEncoder, valueEncoder);
}
return null;
};
preload(storageBasePath);
}
private void preload(final Path storageBasePath) {
try {
Files.list(storageBasePath)//
.filter(Files::isDirectory)//
.map(Path::getFileName)//
.map(Path::toString)//
.map(ParititionId::of)//
.forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier));
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private void preload(final Path storageBasePath) {
try {
Files.list(storageBasePath)//
.filter(Files::isDirectory)//
.map(Path::getFileName)//
.map(Path::toString)//
.map(ParititionId::of)//
.forEach(partitionId -> maps.computeIfAbsent(partitionId, supplier));
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
private Set<ParititionId> getAllPartitionIds() {
return maps.keySet();
}
private Set<ParititionId> getAllPartitionIds() {
return maps.keySet();
}
public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) {
return partitionIdSource.toPartitionIds(getAllPartitionIds());
}
public Set<ParititionId> getAvailablePartitionIds(final PartitionIdSource partitionIdSource) {
return partitionIdSource.toPartitionIds(getAllPartitionIds());
}
private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, supplier);
}
private PersistentMap<K, P> getExistingPersistentMap(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, supplier);
}
private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, creator);
}
private PersistentMap<K, P> getPersistentMapCreateIfNotExists(final ParititionId partitionId) {
return maps.computeIfAbsent(partitionId, creator);
}
public V getValue(final ParititionId partitionId, final K key) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
final P persistedValue = map != null ? map.getValue(key) : null;
return valueEncoder.decodeValue(partitionId, persistedValue);
}
public V getValue(final ParititionId partitionId, final K key) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
final P persistedValue = map != null ? map.getValue(key) : null;
return valueEncoder.decodeValue(partitionId, persistedValue);
}
public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) {
final List<V> result = new ArrayList<>();
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
public List<V> getValues(final PartitionIdSource partitionIdSource, final K key) {
final List<V> result = new ArrayList<>();
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
if (map != null) {
final V value = valueEncoder.decodeValue(partitionId, map.getValue(key));
if (value != null) {
result.add(value);
}
}
}
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
if (map != null) {
final V value = valueEncoder.decodeValue(partitionId, map.getValue(key));
if (value != null) {
result.add(value);
}
}
}
return result;
}
return result;
}
public V putValue(final ParititionId partitionId, final K key, final V value) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
final P persistedValue = valueEncoder.encodeValue(value);
final P previousPersistedValue = map.putValue(key, persistedValue);
return valueEncoder.decodeValue(partitionId, previousPersistedValue);
}
public V putValue(final ParititionId partitionId, final K key, final V value) {
final PersistentMap<K, P> map = getPersistentMapCreateIfNotExists(partitionId);
final P persistedValue = valueEncoder.encodeValue(value);
final P previousPersistedValue = map.putValue(key, persistedValue);
return valueEncoder.decodeValue(partitionId, previousPersistedValue);
}
public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
public void visitValues(final ParititionId partitionId, final K keyPrefix, final Visitor<K, V> visitor) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) {
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
public void visitValues(final PartitionIdSource partitionIdSource, final K keyPrefix, final Visitor<K, V> visitor) {
final Set<ParititionId> partitionIds = partitionIdSource.toPartitionIds(getAllPartitionIds());
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
}
for (final ParititionId partitionId : partitionIds) {
final PersistentMap<K, P> map = getExistingPersistentMap(partitionId);
if (map != null) {
map.visitValues(keyPrefix, (k, p) -> {
final V value = valueEncoder.decodeValue(partitionId, p);
visitor.visit(k, value);
});
}
}
}
@Override
public void close() {
final List<Throwable> throwables = new ArrayList<>();
@Override
public void close() {
final List<Throwable> throwables = new ArrayList<>();
for (final PersistentMap<K, P> map : maps.values()) {
try {
map.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
for (final PersistentMap<K, P> map : maps.values()) {
try {
map.close();
} catch (final RuntimeException e) {
throwables.add(e);
}
}
if (!throwables.isEmpty()) {
final RuntimeException ex = new RuntimeException();
throwables.forEach(ex::addSuppressed);
throw ex;
}
}
}

88
data-store/src/main/java/org/lucares/pdb/datastore/internal/PdbWriter.java
View File

@@ -17,62 +17,62 @@ import org.slf4j.LoggerFactory;
*/
class PdbWriter implements AutoCloseable, Flushable {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
private final PdbFile pdbFile;
private long lastEpochMilli;
private final PdbFile pdbFile;
private long lastEpochMilli;
private final TimeSeriesFile timeSeriesFile;
private final TimeSeriesFile timeSeriesFile;
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) {
this.pdbFile = pdbFile;
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) {
this.pdbFile = pdbFile;
timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct?
timeSeriesFile = TimeSeriesFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = timeSeriesFile.getLastValue(); // TODO is this last value correct?
lastEpochMilli = optionalLastValue.orElse(0L);
}
lastEpochMilli = optionalLastValue.orElse(0L);
}
public PdbFile getPdbFile() {
return pdbFile;
}
public PdbFile getPdbFile() {
return pdbFile;
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException {
try {
timeSeriesFile.appendTimeValue(epochMilli, value);
public void write(final long epochMilli, final long value) throws WriteException, InvalidValueException {
try {
timeSeriesFile.appendTimeValue(epochMilli, value);
lastEpochMilli = epochMilli;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
lastEpochMilli = epochMilli;
} catch (final RuntimeException e) {
throw new WriteException(e);
}
}
@Override
public void close() {
@Override
public void close() {
LOGGER.debug("close PdbWriter {}", pdbFile);
timeSeriesFile.close();
}
LOGGER.debug("close PdbWriter {}", pdbFile);
timeSeriesFile.close();
}
@Override
public void flush() {
timeSeriesFile.flush();
}
@Override
public void flush() {
timeSeriesFile.flush();
}
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) {
writer.write(entry.getEpochMilli(), entry.getValue());
}
}
}
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) {
writer.write(entry.getEpochMilli(), entry.getValue());
}
}
}
@Override
public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";
}
@Override
public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";
}
}

506
data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java
View File

@@ -143,321 +143,321 @@ import org.lucares.utils.byteencoder.VariableByteEncoder;
*
*/
public class QueryCompletionIndex implements AutoCloseable {
private static final class TwoTags {
private final Tag tagA;
private final Tag tagB;
private static final class TwoTags {
private final Tag tagA;
private final Tag tagB;
public TwoTags(final Tag tagA, final Tag tagB) {
this.tagA = tagA;
this.tagB = tagB;
}
public TwoTags(final Tag tagA, final Tag tagB) {
this.tagA = tagA;
this.tagB = tagB;
}
public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) {
public TwoTags(final String fieldB, final String fieldA, final String valueA, final String valueB) {
tagA = new Tag(fieldA, valueA);
tagB = new Tag(fieldB, valueB);
}
tagA = new Tag(fieldA, valueA);
tagB = new Tag(fieldB, valueB);
}
public Tag getTagA() {
return tagA;
}
public Tag getTagA() {
return tagA;
}
public Tag getTagB() {
return tagB;
}
public Tag getTagB() {
return tagB;
}
@Override
public String toString() {
return tagA + "::" + tagB;
}
}
@Override
public String toString() {
return tagA + "::" + tagB;
}
}
public static final class FieldField {
private final int fieldA;
private final int fieldB;
public static final class FieldField {
private final int fieldA;
private final int fieldB;
public FieldField(final int fieldA, final int fieldB) {
this.fieldA = fieldA;
this.fieldB = fieldB;
}
public FieldField(final int fieldA, final int fieldB) {
this.fieldA = fieldA;
this.fieldB = fieldB;
}
public int getFieldA() {
return fieldA;
}
public int getFieldA() {
return fieldA;
}
public int getFieldB() {
return fieldB;
}
public int getFieldB() {
return fieldB;
}
@Override
public String toString() {
return fieldA + "::" + fieldB;
}
}
@Override
public String toString() {
return fieldA + "::" + fieldB;
}
}
private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> {
private static final class EncoderTwoTags implements EncoderDecoder<TwoTags> {
@Override
public byte[] encode(final TwoTags tagAndField) {
final LongList tmp = new LongList(4);
final Tag tagA = tagAndField.getTagA();
final Tag tagB = tagAndField.getTagB();
@Override
public byte[] encode(final TwoTags tagAndField) {
final LongList tmp = new LongList(4);
final Tag tagA = tagAndField.getTagA();
final Tag tagB = tagAndField.getTagB();
tmp.add(tagB.getKey());
tmp.add(tagA.getKey());
tmp.add(tagB.getKey());
tmp.add(tagA.getKey());
if (tagA.getValue() >= 0) {
tmp.add(tagA.getValue());
if (tagA.getValue() >= 0) {
tmp.add(tagA.getValue());
// A query for tagA.key and tagA.value and tagB.key is done by setting
// tagB.value==-1.
// The query is then executed as a prefix search. Thus tagB.value must not be
// part of the byte array that is returned.
if (tagB.getValue() >= 0) {
tmp.add(tagB.getValue());
}
} else {
Preconditions.checkSmaller(tagB.getValue(), 0,
"if no value for tagA is given, then tagB must also be empty");
}
// A query for tagA.key and tagA.value and tagB.key is done by setting
// tagB.value==-1.
// The query is then executed as a prefix search. Thus tagB.value must not be
// part of the byte array that is returned.
if (tagB.getValue() >= 0) {
tmp.add(tagB.getValue());
}
} else {
Preconditions.checkSmaller(tagB.getValue(), 0,
"if no value for tagA is given, then tagB must also be empty");
}
return VariableByteEncoder.encode(tmp);
}
return VariableByteEncoder.encode(tmp);
}
@Override
public TwoTags decode(final byte[] bytes) {
@Override
public TwoTags decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int tagBKey = (int) tmp.get(0);
final int tagAKey = (int) tmp.get(1);
final int tagAValue = (int) tmp.get(2);
final int tagBValue = (int) tmp.get(3);
final LongList tmp = VariableByteEncoder.decode(bytes);
final int tagBKey = (int) tmp.get(0);
final int tagAKey = (int) tmp.get(1);
final int tagAValue = (int) tmp.get(2);
final int tagBValue = (int) tmp.get(3);
final Tag tagA = new Tag(tagAKey, tagAValue);
final Tag tagB = new Tag(tagBKey, tagBValue);
final Tag tagA = new Tag(tagAKey, tagAValue);
final Tag tagB = new Tag(tagBKey, tagBValue);
return new TwoTags(tagA, tagB);
}
return new TwoTags(tagA, tagB);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0, 0, 0, 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0, 0, 0, 0 };
}
}
private static final class EncoderTag implements EncoderDecoder<Tag> {
private static final class EncoderTag implements EncoderDecoder<Tag> {
@Override
public byte[] encode(final Tag tag) {
@Override
public byte[] encode(final Tag tag) {
final LongList longList = new LongList(2);
longList.add(tag.getKey());
final LongList longList = new LongList(2);
longList.add(tag.getKey());
if (tag.getValue() >= 0) {
longList.add(tag.getValue());
}
return VariableByteEncoder.encode(longList);
}
if (tag.getValue() >= 0) {
longList.add(tag.getValue());
}
return VariableByteEncoder.encode(longList);
}
@Override
public Tag decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int key = (int) tmp.get(0);
final int value = (int) tmp.get(1);
return new Tag(key, value);
}
@Override
public Tag decode(final byte[] bytes) {
final LongList tmp = VariableByteEncoder.decode(bytes);
final int key = (int) tmp.get(0);
final int value = (int) tmp.get(1);
return new Tag(key, value);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private static final class EncoderField implements EncoderDecoder<String> {
private static final class EncoderField implements EncoderDecoder<String> {
@Override
public byte[] encode(final String field) {
@Override
public byte[] encode(final String field) {
if (field.isEmpty()) {
return new byte[0];
}
if (field.isEmpty()) {
return new byte[0];
}
return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field));
}
return VariableByteEncoder.encode(Tags.STRING_COMPRESSOR.put(field));
}
@Override
public String decode(final byte[] bytes) {
final long compressedString = VariableByteEncoder.decodeFirstValue(bytes);
return Tags.STRING_COMPRESSOR.get((int) compressedString);
}
@Override
public String decode(final byte[] bytes) {
final long compressedString = VariableByteEncoder.decodeFirstValue(bytes);
return Tags.STRING_COMPRESSOR.get((int) compressedString);
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}
private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex;
private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex;
private final PartitionPersistentMap<String, Empty, Empty> fieldIndex;
private final PartitionPersistentMap<TwoTags, Empty, Empty> tagToTagIndex;
private final PartitionPersistentMap<Tag, Empty, Empty> fieldToValueIndex;
private final PartitionPersistentMap<String, Empty, Empty> fieldIndex;
public QueryCompletionIndex(final Path basePath) throws IOException {
tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
public QueryCompletionIndex(final Path basePath) throws IOException {
tagToTagIndex = new PartitionPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs",
new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldToValueIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs",
new EncoderTag(), PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
}
fieldIndex = new PartitionPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(),
PartitionAwareWrapper.wrap(PersistentMap.EMPTY_ENCODER));
}
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags();
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags();
// index all combinations of tagA and tagB and fieldA to fieldB
for (final Tag tagA : listOfTagsA) {
for (final Tag tagB : listOfTagsB) {
final TwoTags key = new TwoTags(tagA, tagB);
tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE);
}
}
// index all combinations of tagA and tagB and fieldA to fieldB
for (final Tag tagA : listOfTagsA) {
for (final Tag tagB : listOfTagsB) {
final TwoTags key = new TwoTags(tagA, tagB);
tagToTagIndex.putValue(partitionId, key, Empty.INSTANCE);
}
}
// create indices of all tags and all fields
for (final Tag tag : listOfTagsA) {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE);
}
}
// create indices of all tags and all fields
for (final Tag tag : listOfTagsA) {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, tag.getKeyAsString(), Empty.INSTANCE);
}
}
@Override
public void close() throws IOException {
tagToTagIndex.close();
}
@Override
public void close() throws IOException {
tagToTagIndex.close();
}
/**
* Find values for fieldB that are yield results when executing the query
* "fieldA=valueA and fieldB=???"
*
* @param dateRange the date range
* @param fieldA the other field of the and expression
* @param valueA {@link GlobMatcher} for the value of the other field
* @param fieldB the field we are searching values for
* @return values of fieldB
*/
public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA,
final String fieldB) {
/**
* Find values for fieldB that are yield results when executing the query
* "fieldA=valueA and fieldB=???"
*
* @param dateRange the date range
* @param fieldA the other field of the and expression
* @param valueA {@link GlobMatcher} for the value of the other field
* @param fieldB the field we are searching values for
* @return values of fieldB
*/
public SortedSet<String> find(final DateTimeRange dateRange, final String fieldA, final GlobMatcher valueA,
final String fieldB) {
final SortedSet<String> result = new TreeSet<>();
final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null);
final TwoTags keyPrefix = new TwoTags(fieldB, fieldA, null, null);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final String vA = k.getTagA().getValueAsString();
final String vA = k.getTagA().getValueAsString();
if (valueA.matches(vA)) {
result.add(k.getTagB().getValueAsString());
}
});
if (valueA.matches(vA)) {
result.add(k.getTagB().getValueAsString());
}
});
return result;
}
return result;
}
/**
* Find values for fieldB that are yield results when executing the query
* "tag.field=tag.value and fieldB=???"
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return values for the field
*/
public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) {
/**
* Find values for fieldB that are yield results when executing the query
* "tag.field=tag.value and fieldB=???"
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return values for the field
*/
public SortedSet<String> find(final DateTimeRange dateRange, final Tag tag, final String field) {
final SortedSet<String> result = new TreeSet<>();
final int tagBKey = Tags.STRING_COMPRESSOR.put(field);
final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See
// EncoderTwoTags
final TwoTags keyPrefix = new TwoTags(tag, tagB);
final SortedSet<String> result = new TreeSet<>();
final int tagBKey = Tags.STRING_COMPRESSOR.put(field);
final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See
// EncoderTwoTags
final TwoTags keyPrefix = new TwoTags(tag, tagB);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getTagB().getValueAsString());
});
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getTagB().getValueAsString());
});
return result;
}
return result;
}
/**
* Find all values for the given field.
*
* @param dateRange the date range
* @param field the field
* @return the values
*/
public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) {
/**
* Find all values for the given field.
*
* @param dateRange the date range
* @param field the field
* @return the values
*/
public SortedSet<String> findAllValuesForField(final DateTimeRange dateRange, final String field) {
final SortedSet<String> result = new TreeSet<>();
final int tagKey = Tags.STRING_COMPRESSOR.put(field);
final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See
final SortedSet<String> result = new TreeSet<>();
final int tagKey = Tags.STRING_COMPRESSOR.put(field);
final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getValueAsString());
});
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldToValueIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
result.add(k.getValueAsString());
});
return result;
}
return result;
}
/**
* Find values for {@code field} that will yield results for the query
* "tag.field=tag.value and not field=???".
* <p>
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return the values
*/
public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag,
final String field) {
final SortedSet<String> result = new TreeSet<>();
/**
* Find values for {@code field} that will yield results for the query
* "tag.field=tag.value and not field=???".
* <p>
*
* @param dateRange the date range
* @param tag the other tag
* @param field the field we are searching values for
* @return the values
*/
public SortedSet<String> findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag,
final String field) {
final SortedSet<String> result = new TreeSet<>();
final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null);
final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null);
final int negatedValueA = tag.getValue();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final int negatedValueA = tag.getValue();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
tagToTagIndex.visitValues(partitionIdSource, keyPrefix, (k, v) -> {
final int valueA = k.getTagA().getValue();
if (valueA != negatedValueA) {
result.add(k.getTagB().getValueAsString());
}
});
final int valueA = k.getTagA().getValue();
if (valueA != negatedValueA) {
result.add(k.getTagB().getValueAsString());
}
});
return result;
}
return result;
}
public SortedSet<String> findAllFields(final DateTimeRange dateRange) {
final SortedSet<String> result = new TreeSet<>();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
result.add(k);
});
return result;
}
public SortedSet<String> findAllFields(final DateTimeRange dateRange) {
final SortedSet<String> result = new TreeSet<>();
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
result.add(k);
});
return result;
}
public boolean hasField(final DateTimeRange dateRange, final String field) {
final AtomicBoolean found = new AtomicBoolean(false);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
if (k.equals(field)) {
found.set(true);
}
});
return found.get();
}
public boolean hasField(final DateTimeRange dateRange, final String field) {
final AtomicBoolean found = new AtomicBoolean(false);
final PartitionIdSource partitionIdSource = new DatePartitioner(dateRange);
fieldIndex.visitValues(partitionIdSource, "", (k, v) -> {
if (k.equals(field)) {
found.set(true);
}
});
return found.get();
}
}

92
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagEncoderDecoder.java
View File

@@ -7,58 +7,58 @@ import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
import org.lucares.utils.byteencoder.VariableByteEncoder;
class TagEncoderDecoder implements EncoderDecoder<Tag> {
@Override
public byte[] encode(final Tag tag) {
@Override
public byte[] encode(final Tag tag) {
final LongList keyAndValueCompressed = new LongList(2);
final LongList keyAndValueCompressed = new LongList(2);
final String key = tag.getKeyAsString();
final byte[] result;
if (!key.isEmpty()) {
final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key);
keyAndValueCompressed.add(keyAsLong);
final String key = tag.getKeyAsString();
final byte[] result;
if (!key.isEmpty()) {
final Integer keyAsLong = Tags.STRING_COMPRESSOR.put(key);
keyAndValueCompressed.add(keyAsLong);
final String value = tag.getValueAsString();
if (!value.isEmpty()) {
final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value);
keyAndValueCompressed.add(valueAsLong);
}
result = VariableByteEncoder.encode(keyAndValueCompressed);
} else {
result = new byte[0];
}
final String value = tag.getValueAsString();
if (!value.isEmpty()) {
final Integer valueAsLong = Tags.STRING_COMPRESSOR.put(value);
keyAndValueCompressed.add(valueAsLong);
}
result = VariableByteEncoder.encode(keyAndValueCompressed);
} else {
result = new byte[0];
}
return result;
}
return result;
}
@Override
public Tag decode(final byte[] bytes) {
final LongList compressedStrings = VariableByteEncoder.decode(bytes);
final Tag result;
switch (compressedStrings.size()) {
case 0:
@Override
public Tag decode(final byte[] bytes) {
final LongList compressedStrings = VariableByteEncoder.decode(bytes);
final Tag result;
switch (compressedStrings.size()) {
case 0:
result = new Tag("", "");
break;
case 1:
final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
result = new Tag(k, "");
result = new Tag("", "");
break;
case 1:
final String k = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
result = new Tag(k, "");
break;
case 2:
final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1));
result = new Tag(key, value);
break;
default:
throw new IllegalStateException("too many values: " + compressedStrings);
}
break;
case 2:
final String key = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(0));
final String value = Tags.STRING_COMPRESSOR.get((int) compressedStrings.get(1));
result = new Tag(key, value);
break;
default:
throw new IllegalStateException("too many values: " + compressedStrings);
}
return result;
}
@Override
public byte[] getEmptyValue() {
return new byte[] {0};
}
return result;
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}
}

26
data-store/src/main/java/org/lucares/pdb/datastore/internal/TagsEncoderDecoder.java
View File

@@ -4,18 +4,18 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
class TagsEncoderDecoder implements EncoderDecoder<Tags> {
@Override
public byte[] encode(final Tags tags) {
return tags.toBytes();
}
@Override
public byte[] encode(final Tags tags) {
return tags.toBytes();
}
@Override
public Tags decode(final byte[] bytes) {
return Tags.fromBytes(bytes);
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
@Override
public Tags decode(final byte[] bytes) {
return Tags.fromBytes(bytes);
}
@Override
public byte[] getEmptyValue() {
return new byte[] {};
}
}

64
data-store/src/main/java/org/lucares/pdb/datastore/lang/CandidateGrouper.java
View File

@@ -9,43 +9,43 @@ import java.util.TreeSet;
import java.util.regex.Pattern;
public class CandidateGrouper {
public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) {
public SortedSet<String> group(final Collection<String> values, final String queryWithCaretMarker) {
final TreeSet<String> result = new TreeSet<>();
final int numDotsInValue = countDotsInValue(queryWithCaretMarker);
final TreeSet<String> result = new TreeSet<>();
final int numDotsInValue = countDotsInValue(queryWithCaretMarker);
for (final String value : values) {
// keep everything up to the (numDotsInValue+1)-th
final String[] token = value.split(Pattern.quote("."));
final List<String> tokenlist = new ArrayList<>(Arrays.asList(token));
final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1);
String shortenedValue = String.join(".", prefix);
if (tokenlist.size() > numDotsInValue + 1) {
shortenedValue += ".";
}
result.add(shortenedValue);
}
for (final String value : values) {
// keep everything up to the (numDotsInValue+1)-th
final String[] token = value.split(Pattern.quote("."));
final List<String> tokenlist = new ArrayList<>(Arrays.asList(token));
final List<String> prefix = tokenlist.subList(0, numDotsInValue + 1);
String shortenedValue = String.join(".", prefix);
if (tokenlist.size() > numDotsInValue + 1) {
shortenedValue += ".";
}
result.add(shortenedValue);
}
return result;
}
return result;
}
private int countDotsInValue(final String queryWithCaretMarker) {
private int countDotsInValue(final String queryWithCaretMarker) {
int count = 0;
int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1;
final String delimiter = " (),=!";
int count = 0;
int index = queryWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER) - 1;
final String delimiter = " (),=!";
while (index >= 0) {
final char c = queryWithCaretMarker.charAt(index);
if (delimiter.indexOf(c) >= 0) {
break;
}
if (c == '.') {
count++;
}
index--;
}
while (index >= 0) {
final char c = queryWithCaretMarker.charAt(index);
if (delimiter.indexOf(c) >= 0) {
break;
}
if (c == '.') {
count++;
}
index--;
}
return count;
}
return count;
}
}

18
data-store/src/main/java/org/lucares/pdb/datastore/lang/ErrorListener.java
View File

@@ -6,14 +6,14 @@ import org.antlr.v4.runtime.Recognizer;
public class ErrorListener extends BaseErrorListener {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
final int lineStart = line;
final int startIndex = charPositionInLine;
final int lineStop = line;
final int stopIndex = charPositionInLine;
throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex);
}
final int lineStart = line;
final int startIndex = charPositionInLine;
final int lineStop = line;
final int stopIndex = charPositionInLine;
throw new SyntaxException(msg, lineStart, startIndex, lineStop, stopIndex);
}
}

1426
data-store/src/main/java/org/lucares/pdb/datastore/lang/Expression.java
View File

File diff suppressed because it is too large Load Diff

263
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java
View File

@@ -26,179 +26,180 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ExpressionToDocIdVisitor extends ExpressionVisitor<PartitionLongList> {
private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class);
private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionToDocIdVisitor.class);
private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId;
private final PartitionDiskStore diskStorage;
private final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocId;
private final PartitionDiskStore diskStorage;
private final DatePartitioner datePartitioner;
private final DatePartitioner datePartitioner;
public ExpressionToDocIdVisitor(final DateTimeRange dateRange,
final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) {
this.datePartitioner = new DatePartitioner(dateRange);
this.keyToValueToDocId = keyToValueToDocsId;
this.diskStorage = diskStorage;
}
public ExpressionToDocIdVisitor(final DateTimeRange dateRange,
final PartitionPersistentMap<Tag, Long, Long> keyToValueToDocsId, final PartitionDiskStore diskStorage) {
this.datePartitioner = new DatePartitioner(dateRange);
this.keyToValueToDocId = keyToValueToDocsId;
this.diskStorage = diskStorage;
}
@Override
public PartitionLongList visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public PartitionLongList visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles);
LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.intersection(leftFiles, rightFiles);
LOGGER.trace("and: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public PartitionLongList visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles);
LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
final PartitionLongList leftFiles = left.visit(this);
final PartitionLongList rightFiles = right.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = PartitionLongList.union(leftFiles, rightFiles);
LOGGER.trace("or: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
assert result.isSorted();
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Not expression) {
@Override
public PartitionLongList visit(final Not expression) {
final Expression negatedExpression = expression.getExpression();
final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this);
final long start = System.nanoTime();
final Expression negatedExpression = expression.getExpression();
final PartitionLongList docIdsToBeNegated = negatedExpression.visit(this);
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
result.removeAll(docIdsToBeNegated);
final PartitionLongList result = getAllDocIds();
result.removeAll(docIdsToBeNegated);
LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
LOGGER.trace("not: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
return result;
}
@Override
public PartitionLongList visit(final Parentheses parentheses) {
@Override
public PartitionLongList visit(final Parentheses parentheses) {
throw new UnsupportedOperationException(
"Parenthesis not supported. The correct order should come from the parser.");
}
throw new UnsupportedOperationException(
"Parenthesis not supported. The correct order should come from the parser.");
}
@Override
public PartitionLongList visit(final Expression.MatchAll expression) {
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
@Override
public PartitionLongList visit(final Expression.MatchAll expression) {
final long start = System.nanoTime();
final PartitionLongList result = getAllDocIds();
LOGGER.trace("matchAll: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
@Override
public PartitionLongList visit(final Expression.InExpression expression) {
final long start = System.nanoTime();
@Override
public PartitionLongList visit(final Expression.InExpression expression) {
final long start = System.nanoTime();
final String propertyName = expression.getProperty();
final List<String> values = expression.getValues();
final String propertyName = expression.getProperty();
final List<String> values = expression.getValues();
PartitionLongList result = new PartitionLongList();
PartitionLongList result = new PartitionLongList();
for (final String value : values) {
for (final String value : values) {
final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value));
result = PartitionLongList.union(result, docIds);
}
final PartitionLongList docIds = filterByWildcard(propertyName, GloblikePattern.globlikeToRegex(value));
result = PartitionLongList.union(result, docIds);
}
LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
LOGGER.trace("in: {} took {} ms results={}", expression, (System.nanoTime() - start) / 1_000_000.0,
result.size());
return result;
}
private PartitionLongList getAllDocIds() {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
private PartitionLongList getAllDocIds() {
final PartitionLongList result = new PartitionLongList();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);
final LongList tmp = bsFile.asLongList();
result.put(partitionId, tmp);
}
}
return result;
}
if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);
final LongList tmp = bsFile.asLongList();
result.put(partitionId, tmp);
}
}
return result;
}
private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) {
final PartitionLongList result = new PartitionLongList();
private PartitionLongList filterByWildcard(final String propertyName, final Pattern valuePattern) {
final PartitionLongList result = new PartitionLongList();
final long start = System.nanoTime();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final List<LongList> docIdsForPartition = new ArrayList<>();
keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> {
if (valuePattern.matcher(tags.getValueAsString()).matches()) {
try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds, partitionId)) {
final long start = System.nanoTime();
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final List<LongList> docIdsForPartition = new ArrayList<>();
keyToValueToDocId.visitValues(partitionId, new Tag(propertyName, ""), (tags, blockOffsetToDocIds) -> {
if (valuePattern.matcher(tags.getValueAsString()).matches()) {
try (final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffsetToDocIds,
partitionId)) {
// We know that all LongLists coming from a BSFile are sorted, non-overlapping
// and increasing, that means we can just concatenate them and get a sorted
// list.
final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList());
final LongList concatenatedLists = concatenateLists(longLists);
// We know that all LongLists coming from a BSFile are sorted, non-overlapping
// and increasing, that means we can just concatenate them and get a sorted
// list.
final List<LongList> longLists = bsFile.streamOfLongLists().collect(Collectors.toList());
final LongList concatenatedLists = concatenateLists(longLists);
Preconditions.checkTrue(concatenatedLists.isSorted(),
"The LongLists containing document ids must be sorted, "
+ "non-overlapping and increasing, so that the concatenation "
+ "is sorted. This is guaranteed by the fact that document ids "
+ "are generated in monotonically increasing order.");
Preconditions.checkTrue(concatenatedLists.isSorted(),
"The LongLists containing document ids must be sorted, "
+ "non-overlapping and increasing, so that the concatenation "
+ "is sorted. This is guaranteed by the fact that document ids "
+ "are generated in monotonically increasing order.");
docIdsForPartition.add(concatenatedLists);
}
}
});
docIdsForPartition.add(concatenatedLists);
}
}
});
final LongList mergedDocsIdsForPartition = merge(docIdsForPartition);
result.put(partitionId, mergedDocsIdsForPartition);
}
final LongList mergedDocsIdsForPartition = merge(docIdsForPartition);
result.put(partitionId, mergedDocsIdsForPartition);
}
LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0);
LOGGER.trace("filterByWildcard: for key {} took {}ms", propertyName, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
return result;
}
private LongList merge(final Collection<LongList> lists) {
private LongList merge(final Collection<LongList> lists) {
LongList result = new LongList();
LongList result = new LongList();
for (final LongList list : lists) {
result = LongList.union(result, list);
}
for (final LongList list : lists) {
result = LongList.union(result, list);
}
return result;
}
return result;
}
private static LongList concatenateLists(final Collection<LongList> lists) {
private static LongList concatenateLists(final Collection<LongList> lists) {
final int totalSize = lists.stream().mapToInt(LongList::size).sum();
final LongList result = new LongList(totalSize);
final int totalSize = lists.stream().mapToInt(LongList::size).sum();
final LongList result = new LongList(totalSize);
for (final LongList list : lists) {
result.addAll(list);
}
for (final LongList list : lists) {
result.addAll(list);
}
return result;
return result;
}
}
}

66
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionVisitor.java
View File

@@ -1,47 +1,47 @@
package org.lucares.pdb.datastore.lang;
public abstract class ExpressionVisitor<T> {
public T visit(final Expression.And expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.And expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.InExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.InExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndNotCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.AndNotCaretExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.CaretAndExpression expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.CaretAndExpression expression) {
throw new UnsupportedOperationException();
}
}

436
data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java
View File

@@ -22,278 +22,278 @@ import org.slf4j.LoggerFactory;
public class FindValuesForQueryCompletion extends ExpressionVisitor<SortedSet<String>> {
private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret");
private static final Logger METRIC_AND_CARET_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation.andCaret");
private static final Logger METRIC_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation");
private static final Logger METRIC_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.queryCompletion.expressionEvaluation");
private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> {
private final QueryCompletionIndex index;
private final String field;
private final DateTimeRange dateTimeRange;
private static final class AndCaretExpressionVisitor extends ExpressionVisitor<SortedSet<String>> {
private final QueryCompletionIndex index;
private final String field;
private final DateTimeRange dateTimeRange;
public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange,
final QueryCompletionIndex queryCompletionIndex, final String field) {
this.dateTimeRange = dateTimeRange;
index = queryCompletionIndex;
this.field = field;
}
public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange,
final QueryCompletionIndex queryCompletionIndex, final String field) {
this.dateTimeRange = dateTimeRange;
index = queryCompletionIndex;
this.field = field;
}
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final SortedSet<String> result;
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = property.getField();
final String valueA = property.getValue().getValue();
final String fieldA = property.getField();
final String valueA = property.getValue().getValue();
final boolean hasField = index.hasField(dateTimeRange, fieldA);
if (hasField) {
final boolean hasField = index.hasField(dateTimeRange, fieldA);
if (hasField) {
final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA);
final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new);
final SortedSet<String> allValuesForField = index.findAllValuesForField(dateTimeRange, fieldA);
final SortedSet<String> valuesA = GloblikePattern.filterValues(allValuesForField, valueA, TreeSet::new);
final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size();
final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was
// chosen
// arbitrarily
if (useMultiFetch) {
result = new TreeSet<>();
final double valueInFieldAMatchPercentage = valuesA.size() / (double) allValuesForField.size();
final boolean useMultiFetch = valuesA.size() <= 1 || valueInFieldAMatchPercentage < 0.5; // 50% was
// chosen
// arbitrarily
if (useMultiFetch) {
result = new TreeSet<>();
for (final String v : valuesA) {
final Tag tagA = new Tag(fieldA, v);
final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field);
result.addAll(tmp);
}
} else {
result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field);
}
for (final String v : valuesA) {
final Tag tagA = new Tag(fieldA, v);
final SortedSet<String> tmp = index.find(dateTimeRange, tagA, field);
result.addAll(tmp);
}
} else {
result = index.find(dateTimeRange, fieldA, new GlobMatcher(valueA), field);
}
METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)",
useMultiFetch ? "multi-fetch" : "single-fetch", property, field,
(System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100);
METRIC_AND_CARET_LOGGER.debug("{}: {} and {}=???: {}ms matches in fieldA {} ({}%)",
useMultiFetch ? "multi-fetch" : "single-fetch", property, field,
(System.nanoTime() - start) / 1_000_000.0, valuesA.size(), valueInFieldAMatchPercentage * 100);
} else {
result = new TreeSet<>();
}
return result;
}
} else {
result = new TreeSet<>();
}
return result;
}
@Override
public SortedSet<String> visit(final InExpression expression) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = expression.getProperty();
final List<String> values = expression.getValues();
@Override
public SortedSet<String> visit(final InExpression expression) {
final long start = System.nanoTime();
final SortedSet<String> result;
final String fieldA = expression.getProperty();
final List<String> values = expression.getValues();
result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field);
result = index.find(dateTimeRange, fieldA, new GlobMatcher(values), field);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
try {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
try {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left instanceof Property && right instanceof Not) {
final Property leftProperty = (Property) left;
if (left instanceof Property && right instanceof Not) {
final Property leftProperty = (Property) left;
final SortedSet<String> allValuesForField = leftProperty.visit(this);
final SortedSet<String> allValuesForField = leftProperty.visit(this);
final Expression rightInnerExpression = ((Not) right).getExpression();
final SortedSet<String> rightResult = rightInnerExpression.visit(this);
final Expression rightInnerExpression = ((Not) right).getExpression();
final SortedSet<String> rightResult = rightInnerExpression.visit(this);
return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new);
return CollectionUtils.removeAll(allValuesForField, rightResult, TreeSet::new);
} else {
} else {
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult);
result.retainAll(rightResult);
return result;
}
} finally {
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
}
}
return result;
}
} finally {
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
}
}
@Override
public SortedSet<String> visit(final Or expression) {
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult);
result.addAll(rightResult);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Not expression) {
@Override
public SortedSet<String> visit(final Not expression) {
final long start = System.nanoTime();
if (!(expression.getExpression() instanceof Property)) {
throw new UnsupportedOperationException("NOT expressions like '" + expression
+ "' are not supported. Only 'NOT property=value' expressions are supported.");
}
final long start = System.nanoTime();
if (!(expression.getExpression() instanceof Property)) {
throw new UnsupportedOperationException("NOT expressions like '" + expression
+ "' are not supported. Only 'NOT property=value' expressions are supported.");
}
final Property property = (Property) expression.getExpression();
final Tag tag = new Tag(property.getField(), property.getValueAsString());
final Property property = (Property) expression.getExpression();
final Tag tag = new Tag(property.getField(), property.getValueAsString());
final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field);
final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field);
final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField,
valuesNotForField, TreeSet::new);
final SortedSet<String> valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field);
final SortedSet<String> valuesForField = index.find(dateTimeRange, tag, field);
final SortedSet<String> valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField,
valuesNotForField, TreeSet::new);
final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField,
TreeSet::new);
final SortedSet<String> result = CollectionUtils.removeAll(valuesNotForField, valuesOnlyAvailableInField,
TreeSet::new);
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}
private final QueryCompletionIndex queryCompletionIndex;
private final DateTimeRange dateRange;
public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final QueryCompletionIndex queryCompletionIndex) {
this.dateRange = dateRange;
this.queryCompletionIndex = queryCompletionIndex;
}
@Override
public SortedSet<String> visit(final Property property) {
final long start = System.nanoTime();
final String field = property.getField();
final String value = property.getValue().getValue();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valuePrefix;
if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
private final QueryCompletionIndex queryCompletionIndex;
@Override
public SortedSet<String> visit(final AndCaretExpression expression) {
private final DateTimeRange dateRange;
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
public FindValuesForQueryCompletion(final DateTimeRange dateRange,
final QueryCompletionIndex queryCompletionIndex) {
this.dateRange = dateRange;
this.queryCompletionIndex = queryCompletionIndex;
}
final Expression rightHandExpression = expression.getExpression();
@Override
public SortedSet<String> visit(final Property property) {
final SortedSet<String> candidateValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
final long start = System.nanoTime();
final String field = property.getField();
final String value = property.getValue().getValue();
final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new);
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final String valuePrefix;
@Override
public SortedSet<String> visit(final AndNotCaretExpression expression) {
if (value.indexOf(NewProposerParser.CARET_MARKER) >= 0) {
valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER));
} else {
valuePrefix = value;
}
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange,
caretExpression.getField());
final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
valuePattern, TreeSet::new);
@Override
public SortedSet<String> visit(final AndCaretExpression expression) {
final Expression rightHandExpression = expression.getExpression();
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final SortedSet<String> rightHandValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
final Expression rightHandExpression = expression.getExpression();
if (rightHandValues.size() == 1) {
// there is only one alternative and that one must not be chosen
return Collections.emptySortedSet();
}
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> candidateValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
@Override
public SortedSet<String> visit(final Not expression) {
final TreeSet<String> result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new);
final String field;
final Expression innerExpression = expression.getExpression();
if (innerExpression instanceof Property) {
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public SortedSet<String> visit(final AndNotCaretExpression expression) {
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final long start = System.nanoTime();
final Property caretExpression = expression.getCaretExpression();
final String field = caretExpression.getField();
final String valueWithCaretMarker = caretExpression.getValue().getValue();
final String valuePattern = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
result.addAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange,
caretExpression.getField());
final SortedSet<String> valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField,
valuePattern, TreeSet::new);
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression rightHandExpression = expression.getExpression();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
final SortedSet<String> rightHandValues = rightHandExpression
.visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field));
result.retainAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
if (rightHandValues.size() == 1) {
// there is only one alternative and that one must not be chosen
return Collections.emptySortedSet();
}
final SortedSet<String> result = CollectionUtils.retainAll(rightHandValues,
valuesForFieldMatchingCaretExpression, TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final Not expression) {
final String field;
final Expression innerExpression = expression.getExpression();
if (innerExpression instanceof Property) {
final long start = System.nanoTime();
field = ((Property) innerExpression).getField();
final SortedSet<String> allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field);
final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue();
final String valuePrefix = valueWithCaretMarker.substring(0,
valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER));
final TreeSet<String> result = GloblikePattern.filterValues(allValuesForField, valuePrefix + "*",
TreeSet::new);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
} else {
throw new UnsupportedOperationException();
}
}
@Override
public SortedSet<String> visit(final Or expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.addAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@Override
public SortedSet<String> visit(final And expression) {
final long start = System.nanoTime();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final SortedSet<String> result = left.visit(this);
final SortedSet<String> rightResult = right.visit(this);
result.retainAll(rightResult);
METRIC_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0);
return result;
}
}

94
data-store/src/main/java/org/lucares/pdb/datastore/lang/GloblikePattern.java
View File

@@ -12,70 +12,70 @@ import org.slf4j.LoggerFactory;
public class GloblikePattern {
private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class);
private static final Logger LOGGER = LoggerFactory.getLogger(GloblikePattern.class);
enum FilterMode {
KEEP_EQUAL
}
enum FilterMode {
KEEP_EQUAL
}
public static Pattern globlikeToRegex(final String globlike) {
public static Pattern globlikeToRegex(final String globlike) {
final String valueRegex = "^" + globlikeToPattern(globlike);
final String valueRegex = "^" + globlikeToPattern(globlike);
LOGGER.trace(">{}< -> >{}<", globlike, valueRegex);
LOGGER.trace(">{}< -> >{}<", globlike, valueRegex);
return Pattern.compile(valueRegex);
}
return Pattern.compile(valueRegex);
}
public static Pattern globlikeToRegex(final Iterable<String> globlikes) {
public static Pattern globlikeToRegex(final Iterable<String> globlikes) {
final List<String> regex = new ArrayList<>();
final List<String> regex = new ArrayList<>();
for (final String globlike : globlikes) {
regex.add(globlikeToPattern(globlike));
}
final StringBuilder fullRegex = new StringBuilder("^(");
fullRegex.append(String.join("|", regex));
fullRegex.append(")");
for (final String globlike : globlikes) {
regex.add(globlikeToPattern(globlike));
}
final StringBuilder fullRegex = new StringBuilder("^(");
fullRegex.append(String.join("|", regex));
fullRegex.append(")");
LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex);
LOGGER.trace(">{}< -> >{}<", globlikes, fullRegex);
return Pattern.compile(fullRegex.toString());
}
return Pattern.compile(fullRegex.toString());
}
private static String globlikeToPattern(final String globlike) {
// a character that cannot be in the globPattern
final String dotPlaceholder = "\ue003"; // fourth character in the private use area
private static String globlikeToPattern(final String globlike) {
// a character that cannot be in the globPattern
final String dotPlaceholder = "\ue003"; // fourth character in the private use area
final String valueRegex = globlike//
.replace("-", Pattern.quote("-"))//
.replace(".", dotPlaceholder)//
.replace("*", ".*")//
.replace(dotPlaceholder, ".*\\.")//
.replaceAll("([A-Z])", "[a-z]*$1");
return valueRegex;
}
final String valueRegex = globlike//
.replace("-", Pattern.quote("-"))//
.replace(".", dotPlaceholder)//
.replace("*", ".*")//
.replace(dotPlaceholder, ".*\\.")//
.replaceAll("([A-Z])", "[a-z]*$1");
return valueRegex;
}
public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues,
final String valuePattern, final Supplier<T> generator) {
final T result = generator.get();
public static <T extends Collection<String>> T filterValues(final Collection<String> availableValues,
final String valuePattern, final Supplier<T> generator) {
final T result = generator.get();
return filterValues(result, availableValues, valuePattern);
}
return filterValues(result, availableValues, valuePattern);
}
public static <T extends Collection<String>> T filterValues(final T result,
final Collection<String> availableValues, final String valuePattern) {
public static <T extends Collection<String>> T filterValues(final T result,
final Collection<String> availableValues, final String valuePattern) {
final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern);
final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern);
for (final String value : availableValues) {
final Matcher matcher = pattern.matcher(value);
if (matcher.find()) {
result.add(value);
}
}
for (final String value : availableValues) {
final Matcher matcher = pattern.matcher(value);
if (matcher.find()) {
result.add(value);
}
}
return result;
}
return result;
}
}

96
data-store/src/main/java/org/lucares/pdb/datastore/lang/IdentityExpressionVisitor.java
View File

@@ -14,66 +14,66 @@ import org.lucares.pdb.datastore.lang.Expression.Property;
* as base class for visitors that modify expressions.
*/
public abstract class IdentityExpressionVisitor extends ExpressionVisitor<Expression> {
@Override
public Expression visit(final And expression) {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
return new And(left, right);
}
return new And(left, right);
}
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft().visit(this);
final Expression right = expression.getRight().visit(this);
return new Or(left, right);
}
return new Or(left, right);
}
@Override
public Expression visit(final Not expression) {
return new Not(expression.getExpression().visit(this));
}
@Override
public Expression visit(final Not expression) {
return new Not(expression.getExpression().visit(this));
}
@Override
public Expression visit(final Property expression) {
return expression;
}
@Override
public Expression visit(final Property expression) {
return expression;
}
@Override
public Expression visit(final Expression.Terminal expression) {
return expression;
}
@Override
public Expression visit(final Expression.Terminal expression) {
return expression;
}
@Override
public Expression visit(final Expression.MatchAll expression) {
return expression;
}
@Override
public Expression visit(final Expression.MatchAll expression) {
return expression;
}
@Override
public Expression visit(final Expression.InExpression expression) {
return expression;
}
@Override
public Expression visit(final Expression.InExpression expression) {
return expression;
}
@Override
public Expression visit(final Parentheses parentheses) {
return new Parentheses(parentheses.getExpression().visit(this));
}
@Override
public Expression visit(final Parentheses parentheses) {
return new Parentheses(parentheses.getExpression().visit(this));
}
@Override
public Expression visit(final AndCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndNotCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final AndNotCaretExpression expression) {
return expression;
}
@Override
public Expression visit(final CaretAndExpression expression) {
return expression;
}
@Override
public Expression visit(final CaretAndExpression expression) {
return expression;
}
}

316
data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java
View File

@@ -21,203 +21,203 @@ import org.slf4j.LoggerFactory;
public class NewProposerParser implements QueryConstants {
private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class);
private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class);
private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose");
private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose");
/*
* Regex matching a java identifier without a caret marker. We define it as a
* blacklist, because this is easer. The regex is only used <em>after</em> the
* query has already been validated with the proper grammar.
*/
private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*";
/*
* Regex matching a java identifier without a caret marker. We define it as a
* blacklist, because this is easer. The regex is only used <em>after</em> the
* query has already been validated with the proper grammar.
*/
private static final String REGEX_IDENTIFIER = "[^\\s,!\\(\\)=" + CARET_MARKER + "]*";
private final QueryCompletionIndex queryCompletionIndex;
private final QueryCompletionIndex queryCompletionIndex;
public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) {
this.queryCompletionIndex = queryCompletionIndex;
}
public NewProposerParser(final QueryCompletionIndex queryCompletionIndex) {
this.queryCompletionIndex = queryCompletionIndex;
}
public List<Proposal> propose(final QueryWithCaretMarker query) {
final long start = System.nanoTime();
List<Proposal> proposals;
if (StringUtils.isBlank(query.getQuery())) {
proposals = proposeForAllKeys(query.getDateRange());
} else {
public List<Proposal> propose(final QueryWithCaretMarker query) {
final long start = System.nanoTime();
List<Proposal> proposals;
if (StringUtils.isBlank(query.getQuery())) {
proposals = proposeForAllKeys(query.getDateRange());
} else {
final List<Proposal> foundProposals = proposalsForValues(query);
if (foundProposals.isEmpty()) {
proposals = proposalsForNonValues(query);
} else {
proposals = foundProposals;
}
}
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
final List<Proposal> foundProposals = proposalsForValues(query);
if (foundProposals.isEmpty()) {
proposals = proposalsForNonValues(query);
} else {
proposals = foundProposals;
}
}
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query);
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query);
return nonEmptyProposals;
}
return nonEmptyProposals;
}
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
/*
* This method is called when the query could not be parsed. It is likely that
* the next word is either a field or an operator. But is is also possible that
* the next word is a field-value, because the syntax error might be at another
* location in the query (not at the caret position).
*/
/*
* This method is called when the query could not be parsed. It is likely that
* the next word is either a field or an operator. But is is also possible that
* the next word is a field-value, because the syntax error might be at another
* location in the query (not at the caret position).
*/
final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker());
final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER));
final List<String> tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker());
final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER));
if (indexTokenWithCaret > 0) {
final String previousToken = tokens.get(indexTokenWithCaret - 1);
switch (previousToken) {
case "(":
case "and":
case "or":
case "!":
proposals.addAll(proposeForAllKeys(query));
break;
if (indexTokenWithCaret > 0) {
final String previousToken = tokens.get(indexTokenWithCaret - 1);
switch (previousToken) {
case "(":
case "and":
case "or":
case "!":
proposals.addAll(proposeForAllKeys(query));
break;
case ")":
default:
// proposals.addAll(proposal);
break;
}
} else if (indexTokenWithCaret == 0) {
proposals.addAll(proposeForAllKeys(query));
}
case ")":
default:
// proposals.addAll(proposal);
break;
}
} else if (indexTokenWithCaret == 0) {
proposals.addAll(proposeForAllKeys(query));
}
return proposals;
}
return proposals;
}
private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker());
private Collection<? extends Proposal> proposeForAllKeys(final QueryWithCaretMarker query) {
final List<Proposal> proposals = new ArrayList<>();
final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker());
if (wordPrefix != null) {
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange());
for (final String field : allFields) {
if (wordPrefix != null) {
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(query.getDateRange());
for (final String field : allFields) {
if (!field.startsWith(wordPrefix)) {
continue;
}
if (!field.startsWith(wordPrefix)) {
continue;
}
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
final String proposedQuery = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* ");
final String newQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER);
final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
}
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
final String proposedQuery = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* ");
final String newQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=" + CARET_MARKER);
final String newQuery = newQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = newQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
}
return proposals;
}
return proposals;
}
private String wordPrefix(final String queryWithCaretMarker) {
private String wordPrefix(final String queryWithCaretMarker) {
final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")");
final Matcher matcher = pattern.matcher(queryWithCaretMarker);
if (matcher.find()) {
final String group = matcher.group();
return group.replace(CARET_MARKER, "");
}
final Pattern pattern = Pattern.compile("(" + REGEX_IDENTIFIER + CARET_MARKER + ")");
final Matcher matcher = pattern.matcher(queryWithCaretMarker);
if (matcher.find()) {
final String group = matcher.group();
return group.replace(CARET_MARKER, "");
}
return null;
}
return null;
}
private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> proposeForAllKeys(final DateTimeRange dateRange) {
final List<Proposal> proposals = new ArrayList<>();
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange);
for (final String field : allFields) {
final String proposedQuery = field + "=*";
final String newQuery = field + "=";
final int newCaretPosition = newQuery.length();
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
final SortedSet<String> allFields = queryCompletionIndex.findAllFields(dateRange);
for (final String field : allFields) {
final String proposedQuery = field + "=*";
final String newQuery = field + "=";
final int newCaretPosition = newQuery.length();
final Proposal proposal = new Proposal(field, proposedQuery, true, newQuery, newCaretPosition);
proposals.add(proposal);
}
return proposals;
}
return proposals;
}
List<Proposal> proposalsForValues(final QueryWithCaretMarker query) {
try {
// Add caret marker, so that we know where the caret is.
// This also makes sure that a query like "name=|" ('|' is the caret) can be
// parsed.
// Without the caret marker the query would be "name=", which is not a valid
// expression.
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
List<Proposal> proposalsForValues(final QueryWithCaretMarker query) {
try {
// Add caret marker, so that we know where the caret is.
// This also makes sure that a query like "name=|" ('|' is the caret) can be
// parsed.
// Without the caret marker the query would be "name=", which is not a valid
// expression.
final String queryWithCaretMarker = query.getQueryWithCaretMarker();
// parse the query
final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker);
// parse the query
final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker);
// normalize it, so that we can use the queryCompletionIndex to search for
// candidate values
final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer();
final Expression normalizedExpression = optimizer.normalizeExpression(expression);
// normalize it, so that we can use the queryCompletionIndex to search for
// candidate values
final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer();
final Expression normalizedExpression = optimizer.normalizeExpression(expression);
// find all candidate values
final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
// find all candidate values
final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker);
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker);
// translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
// translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
return proposals;
} catch (final SyntaxException e) {
LOGGER.debug("Query ({}) is not valid. This is expected to happen "
+ "unless we are looking for proposals of values.", query, e);
return Collections.emptyList();
}
}
return proposals;
} catch (final SyntaxException e) {
LOGGER.debug("Query ({}) is not valid. This is expected to happen "
+ "unless we are looking for proposals of values.", query, e);
return Collections.emptyList();
}
}
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) {
switch (resultMode) {
case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES:
return candidateValues;
default:
throw new IllegalArgumentException("Unexpected value: " + resultMode);
}
}
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) {
switch (resultMode) {
case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES:
return candidateValues;
default:
throw new IllegalArgumentException("Unexpected value: " + resultMode);
}
}
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker);
}
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker);
}
private List<Proposal> generateProposals(final String queryWithCaretMarker,
final SortedSet<String> candidateValues) {
final List<Proposal> proposals = new ArrayList<>();
private List<Proposal> generateProposals(final String queryWithCaretMarker,
final SortedSet<String> candidateValues) {
final List<Proposal> proposals = new ArrayList<>();
for (final String proposedTag : candidateValues) {
for (final String proposedTag : candidateValues) {
final String proposedQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER);
final String proposedQueryWithCaretMarker = queryWithCaretMarker
.replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, proposedTag + CARET_MARKER);
final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER);
final String proposedQuery = proposedQueryWithCaretMarker.replace(CARET_MARKER, "");
final int newCaretPosition = proposedQueryWithCaretMarker.indexOf(CARET_MARKER);
final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition);
proposals.add(proposal);
}
final Proposal proposal = new Proposal(proposedTag, proposedQuery, true, proposedQuery, newCaretPosition);
proposals.add(proposal);
}
return proposals;
}
return proposals;
}
}

368
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionExpressionOptimizer.java
View File

@@ -45,228 +45,228 @@ import org.slf4j.LoggerFactory;
*/
public class QueryCompletionExpressionOptimizer {
private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class);
private static final Logger LOGGER = LoggerFactory.getLogger(QueryCompletionExpressionOptimizer.class);
private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor {
private static final class ReplaceINExpressionsWithPropertyExpressionsVisitor extends IdentityExpressionVisitor {
@Override
public Expression visit(final InExpression expression) {
if (expression.containsCaret() || expression.getValues().size() == 1) {
final String property = expression.getProperty();
final List<String> values = expression.getValues();
@Override
public Expression visit(final InExpression expression) {
if (expression.containsCaret() || expression.getValues().size() == 1) {
final String property = expression.getProperty();
final List<String> values = expression.getValues();
final List<Property> propertyExpressions = new ArrayList<>();
final List<Property> propertyExpressions = new ArrayList<>();
for (final String value : values) {
propertyExpressions.add(new Property(property, new Terminal(value)));
}
for (final String value : values) {
propertyExpressions.add(new Property(property, new Terminal(value)));
}
return Expression.Or.create(propertyExpressions);
} else {
return super.visit(expression);
}
};
}
return Expression.Or.create(propertyExpressions);
} else {
return super.visit(expression);
}
};
}
private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
private static final class RemoveOrEdExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left.containsCaret() && !right.containsCaret()) {
return left;
}
if (!left.containsCaret() && right.containsCaret()) {
return right;
}
return super.visit(expression);
};
}
if (left.containsCaret() && !right.containsCaret()) {
return left;
}
if (!left.containsCaret() && right.containsCaret()) {
return right;
}
return super.visit(expression);
};
}
private static final class DistributiveNormalization extends IdentityExpressionVisitor {
private static final class DistributiveNormalization extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left instanceof Or) {
// (a or b) and c
// becomes
// a and c or b and c
final Expression ac = new And(((Or) left).getLeft(), right);
final Expression bc = new And(((Or) left).getRight(), right);
return new Or(ac, bc);
}
if (left instanceof Or) {
// (a or b) and c
// becomes
// a and c or b and c
final Expression ac = new And(((Or) left).getLeft(), right);
final Expression bc = new And(((Or) left).getRight(), right);
return new Or(ac, bc);
}
if (right instanceof Or) {
// a and (b or c)
// becomes
// a and b or a and c
final Expression ab = new And(left, ((Or) right).getLeft());
final Expression ac = new And(left, ((Or) right).getRight());
return new Or(ab, ac);
}
return super.visit(expression);
};
}
if (right instanceof Or) {
// a and (b or c)
// becomes
// a and b or a and c
final Expression ab = new And(left, ((Or) right).getLeft());
final Expression ac = new And(left, ((Or) right).getRight());
return new Or(ab, ac);
}
return super.visit(expression);
};
}
private static final class RotateAndExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
private static final class RotateAndExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
// (| and a) and b => | and (a and b)
//
// The expression with the caret is moved up
if (left.containsCaret() && left instanceof And) {
final Expression leftLeft = ((And) left).getLeft();
final Expression leftRight = ((And) left).getRight();
// (| and a) and b => | and (a and b)
//
// The expression with the caret is moved up
if (left.containsCaret() && left instanceof And) {
final Expression leftLeft = ((And) left).getLeft();
final Expression leftRight = ((And) left).getRight();
if (leftLeft.containsCaret()) {
return new And(leftLeft, new And(leftRight, right));
} else {
return new And(new And(leftLeft, right), leftRight);
}
} else if (right.containsCaret() && right instanceof And) {
final Expression rightLeft = ((And) right).getLeft();
final Expression rightRight = ((And) right).getRight();
if (leftLeft.containsCaret()) {
return new And(leftLeft, new And(leftRight, right));
} else {
return new And(new And(leftLeft, right), leftRight);
}
} else if (right.containsCaret() && right instanceof And) {
final Expression rightLeft = ((And) right).getLeft();
final Expression rightRight = ((And) right).getRight();
if (rightLeft.containsCaret()) {
return new And(rightLeft, new And(rightRight, left));
} else {
return new And(new And(rightLeft, left), rightRight);
}
}
if (rightLeft.containsCaret()) {
return new And(rightLeft, new And(rightRight, left));
} else {
return new And(new And(rightLeft, left), rightRight);
}
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
private static final class DoubleNegationExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression instanceof Not) {
if (expression.getExpression() instanceof Not) {
return ((Not) expression.getExpression()).getExpression();
}
}
return super.visit(expression);
}
}
private static final class DoubleNegationExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression instanceof Not) {
if (expression.getExpression() instanceof Not) {
return ((Not) expression.getExpression()).getExpression();
}
}
return super.visit(expression);
}
}
private static final class DeMorgan extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
private static final class DeMorgan extends IdentityExpressionVisitor {
@Override
public Expression visit(final Not expression) {
if (expression.getExpression() instanceof And) {
final And andExpression = (And) expression.getExpression();
final Expression left = andExpression.getLeft();
final Expression right = andExpression.getRight();
if (expression.getExpression() instanceof And) {
final And andExpression = (And) expression.getExpression();
final Expression left = andExpression.getLeft();
final Expression right = andExpression.getRight();
final Expression notLeft = new Not(left);
final Expression notRight = new Not(right);
final Expression notLeft = new Not(left);
final Expression notRight = new Not(right);
return new Or(notLeft, notRight);
}
return new Or(notLeft, notRight);
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
private static final class ToAndCaretExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
private static final class ToAndCaretExpressions extends IdentityExpressionVisitor {
@Override
public Expression visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
if (left.containsCaret() && left instanceof Property) {
return new AndCaretExpression((Property) left, right);
}
if (right.containsCaret() && right instanceof Property) {
return new AndCaretExpression((Property) right, left);
}
if (left.containsCaret() && left instanceof Property) {
return new AndCaretExpression((Property) left, right);
}
if (right.containsCaret() && right instanceof Property) {
return new AndCaretExpression((Property) right, left);
}
if (left.containsCaret()//
&& left instanceof Not//
&& ((Not) left).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right);
}
if (right.containsCaret()//
&& right instanceof Not//
&& ((Not) right).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left);
}
if (left.containsCaret()//
&& left instanceof Not//
&& ((Not) left).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) left).getExpression(), right);
}
if (right.containsCaret()//
&& right instanceof Not//
&& ((Not) right).getExpression() instanceof Property) {
return new AndNotCaretExpression((Property) ((Not) right).getExpression(), left);
}
return super.visit(expression);
}
}
return super.visit(expression);
}
}
public Expression normalizeExpression(final Expression expression) {
public Expression normalizeExpression(final Expression expression) {
Expression normalizingExpression = expression;
Expression previousExpression = normalizingExpression;
do {
previousExpression = normalizingExpression;
// replace all IN-expression, because they are just syntactic sugar for
// OR-expressions, but only for those that include the caret
normalizingExpression = normalizingExpression
.visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor());
Expression normalizingExpression = expression;
Expression previousExpression = normalizingExpression;
do {
previousExpression = normalizingExpression;
// replace all IN-expression, because they are just syntactic sugar for
// OR-expressions, but only for those that include the caret
normalizingExpression = normalizingExpression
.visit(new ReplaceINExpressionsWithPropertyExpressionsVisitor());
// Remove expressions that are OR'ed with the one that contains the caret.
// Everything that is OR'ed with the 'caret'-expression cannot change the
// possible values.
normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions());
// Remove expressions that are OR'ed with the one that contains the caret.
// Everything that is OR'ed with the 'caret'-expression cannot change the
// possible values.
normalizingExpression = visitRepeatedly(normalizingExpression, new RemoveOrEdExpressions());
// In the end we want to have expressions like "firstname=Jane and lastname=|".
// To reach that goal we use the distributive law to modify expressions like
// "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and
// lastname=|) or (firstname=John and lastname=|)"
normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization());
// In the end we want to have expressions like "firstname=Jane and lastname=|".
// To reach that goal we use the distributive law to modify expressions like
// "(firstname=Jane or firstname=John) and lastname=|" to "(firstname=Jane and
// lastname=|) or (firstname=John and lastname=|)"
normalizingExpression = visitRepeatedly(normalizingExpression, new DistributiveNormalization());
// (fn=John and (fn=John and ln=|)
// normalized to
// (fn=John and ln=|) and (fn=Jane and ln=|)
// or normalized to
// (fn=John and fn=Jane) and ln=|
normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions());
// (fn=John and (fn=John and ln=|)
// normalized to
// (fn=John and ln=|) and (fn=Jane and ln=|)
// or normalized to
// (fn=John and fn=Jane) and ln=|
normalizingExpression = visitRepeatedly(normalizingExpression, new RotateAndExpressions());
// normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might
// later be removed
// not ( a and b) => (not a) or (not b)
normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan());
// normalize a NAND-expression into an OR with DeMorgan, the OR-Expression might
// later be removed
// not ( a and b) => (not a) or (not b)
normalizingExpression = visitRepeatedly(normalizingExpression, new DeMorgan());
// remove double negation
// not not a => a
normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions());
} while (!normalizingExpression.equals(previousExpression));
// remove double negation
// not not a => a
normalizingExpression = visitRepeatedly(normalizingExpression, new DoubleNegationExpressions());
} while (!normalizingExpression.equals(previousExpression));
// Replaces all (a and |) expressions with a special expression that represents
// it.
// This special expression will then be used during evaluation.
return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions());
}
// Replaces all (a and |) expressions with a special expression that represents
// it.
// This special expression will then be used during evaluation.
return visitRepeatedly(normalizingExpression, new ToAndCaretExpressions());
}
private static Expression visitRepeatedly(final Expression expression,
final ExpressionVisitor<Expression> visitor) {
Expression previousExpression;
Expression result = expression;
private static Expression visitRepeatedly(final Expression expression,
final ExpressionVisitor<Expression> visitor) {
Expression previousExpression;
Expression result = expression;
do {
previousExpression = result;
result = previousExpression.visit(visitor);
if (!previousExpression.equals(result)) {
LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName());
LOGGER.debug(" in: {}", previousExpression);
LOGGER.debug(" out: {}", result);
}
} while (!previousExpression.equals(result));
do {
previousExpression = result;
result = previousExpression.visit(visitor);
if (!previousExpression.equals(result)) {
LOGGER.debug(" translate: {}", visitor.getClass().getSimpleName());
LOGGER.debug(" in: {}", previousExpression);
LOGGER.debug(" out: {}", result);
}
} while (!previousExpression.equals(result));
return result;
}
return result;
}
}

184
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguage.java
View File

@@ -28,125 +28,125 @@ import org.lucares.utils.CollectionUtils;
public class QueryLanguage {
public Expression parse(final String input) {
// define the input
final CharStream in = CharStreams.fromString(input);
public Expression parse(final String input) {
// define the input
final CharStream in = CharStreams.fromString(input);
// create lexer and parser
final PdbLangLexer lexer = new PdbLangLexer(in);
lexer.addErrorListener(new ErrorListener());
// create lexer and parser
final PdbLangLexer lexer = new PdbLangLexer(in);
lexer.addErrorListener(new ErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final PdbLangParser parser = new PdbLangParser(tokens);
parser.addErrorListener(new ErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final PdbLangParser parser = new PdbLangParser(tokens);
parser.addErrorListener(new ErrorListener());
final Stack<Expression> stack = new Stack<>();
final Stack<Expression> stack = new Stack<>();
// define a listener that is called for every terminals and
// non-terminals
final ParseTreeListener listener = new PdbLangBaseListener() {
// define a listener that is called for every terminals and
// non-terminals
final ParseTreeListener listener = new PdbLangBaseListener() {
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
stack.push(new Terminal(ctx.getText()));
}
stack.push(new Terminal(ctx.getText()));
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
stack.push(new Terminal(ctx.getText()));
}
stack.push(new Terminal(ctx.getText()));
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
final Expression expression = stack.pop();
final Expression expression = stack.pop();
final Expression notExpression = new Not(expression);
stack.push(notExpression);
}
final Expression notExpression = new Not(expression);
stack.push(notExpression);
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new AndTemporary();
final Expression left = stack.pop();
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new AndTemporary();
final Expression left = stack.pop();
stack.push(operation.toExpression(left, right));
}
stack.push(operation.toExpression(left, right));
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new OrTemporary();
final Expression left = stack.pop();
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
final Expression right = stack.pop();
final TemporaryExpression operation = new OrTemporary();
final Expression left = stack.pop();
stack.push(operation.toExpression(left, right));
}
stack.push(operation.toExpression(left, right));
}
@Override
public void exitListOfPropValues(final ListOfPropValuesContext ctx) {
final Expression topStackElement = stack.pop();
@Override
public void exitListOfPropValues(final ListOfPropValuesContext ctx) {
final Expression topStackElement = stack.pop();
if (topStackElement instanceof ListOfPropertyValues) {
// there are at least two property values in the query
// e.g. in the expression "bird in (eagle, pigeon)"
final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement;
final Terminal nextPropertyValue = (Terminal) stack.pop();
if (topStackElement instanceof ListOfPropertyValues) {
// there are at least two property values in the query
// e.g. in the expression "bird in (eagle, pigeon)"
final ListOfPropertyValues existingList = (ListOfPropertyValues) topStackElement;
final Terminal nextPropertyValue = (Terminal) stack.pop();
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue,
existingList);
stack.push(newListOfPropertyValues);
} else {
// this is the first or the only value in this list of property values
// e.g. in the expression "bird in (eagle)"
final Terminal propertyValue = (Terminal) topStackElement;
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(nextPropertyValue,
existingList);
stack.push(newListOfPropertyValues);
} else {
// this is the first or the only value in this list of property values
// e.g. in the expression "bird in (eagle)"
final Terminal propertyValue = (Terminal) topStackElement;
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue);
stack.push(newListOfPropertyValues);
}
}
final ListOfPropertyValues newListOfPropertyValues = new ListOfPropertyValues(propertyValue);
stack.push(newListOfPropertyValues);
}
}
@Override
public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) {
@Override
public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) {
final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop();
final Terminal propertyName = (Terminal) stack.pop();
final ListOfPropertyValues propertyValues = (ListOfPropertyValues) stack.pop();
final Terminal propertyName = (Terminal) stack.pop();
final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues());
stack.push(inExpression);
}
};
final InExpression inExpression = new InExpression(propertyName.getValue(), propertyValues.getValues());
stack.push(inExpression);
}
};
// Specify our entry point
final ParseTree parseTree = parser.start();
// Specify our entry point
final ParseTree parseTree = parser.start();
// Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree);
// Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree);
if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack);
}
if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack);
}
return stack.pop();
}
return stack.pop();
}
public static List<String> getTokens(final String input) {
final CharStream in = CharStreams.fromString(input);
public static List<String> getTokens(final String input) {
final CharStream in = CharStreams.fromString(input);
final PdbLangLexer lexer = new PdbLangLexer(in);
final PdbLangLexer lexer = new PdbLangLexer(in);
final CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
final List<Token> tokenList = tokens.getTokens();
return CollectionUtils.map(tokenList, Token::getText);
}
final CommonTokenStream tokens = new CommonTokenStream(lexer);
tokens.fill();
final List<Token> tokenList = tokens.getTokens();
return CollectionUtils.map(tokenList, Token::getText);
}
}

20
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryLanguageParser.java
View File

@@ -3,15 +3,15 @@ package org.lucares.pdb.datastore.lang;
import org.apache.commons.lang3.StringUtils;
public class QueryLanguageParser {
public static Expression parse(final String query) {
public static Expression parse(final String query) {
final Expression result;
if (StringUtils.isEmpty(query)) {
result = Expression.matchAll();
} else {
final QueryLanguage lang = new QueryLanguage();
result = lang.parse(query);
}
return result;
}
final Expression result;
if (StringUtils.isEmpty(query)) {
result = Expression.matchAll();
} else {
final QueryLanguage lang = new QueryLanguage();
result = lang.parse(query);
}
return result;
}
}

90
data-store/src/main/java/org/lucares/pdb/datastore/lang/SyntaxException.java
View File

@@ -4,61 +4,61 @@ import org.antlr.v4.runtime.ParserRuleContext;
public class SyntaxException extends RuntimeException {
private static final long serialVersionUID = 1L;
private int lineStart;
private int startIndex;
private int lineStop;
private int stopIndex;
private static final long serialVersionUID = 1L;
private int lineStart;
private int startIndex;
private int lineStop;
private int stopIndex;
public SyntaxException(final ParserRuleContext context, final String message) {
this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(),
context.getStop().getStopIndex());
}
public SyntaxException(final ParserRuleContext context, final String message) {
this(message, context.getStart().getLine(), context.getStart().getStartIndex(), context.getStop().getLine(),
context.getStop().getStopIndex());
}
public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex));
this.lineStart = lineStart;
this.startIndex = startIndex;
this.lineStop = lineStop;
this.stopIndex = stopIndex;
}
public SyntaxException(final String message, final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
super(message + ": " + generateMessage(lineStart, startIndex, lineStop, stopIndex));
this.lineStart = lineStart;
this.startIndex = startIndex;
this.lineStop = lineStop;
this.stopIndex = stopIndex;
}
private static String generateMessage(final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
private static String generateMessage(final int lineStart, final int startIndex, final int lineStop,
final int stopIndex) {
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
}
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
}
public int getLineStart() {
return lineStart;
}
public int getLineStart() {
return lineStart;
}
public void setLineStart(final int lineStart) {
this.lineStart = lineStart;
}
public void setLineStart(final int lineStart) {
this.lineStart = lineStart;
}
public int getStartIndex() {
return startIndex;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(final int startIndex) {
this.startIndex = startIndex;
}
public void setStartIndex(final int startIndex) {
this.startIndex = startIndex;
}
public int getLineStop() {
return lineStop;
}
public int getLineStop() {
return lineStop;
}
public void setLineStop(final int lineStop) {
this.lineStop = lineStop;
}
public void setLineStop(final int lineStop) {
this.lineStop = lineStop;
}
public int getStopIndex() {
return stopIndex;
}
public int getStopIndex() {
return stopIndex;
}
public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex;
}
public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex;
}
}

460
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -42,293 +42,293 @@ import org.testng.annotations.Test;
@Test
public class DataStoreTest {
private Path dataDirectory;
private DataStore dataStore;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
dataStore = null;
tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null;
}
public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory);
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim));
assertSearch(dateRange, "bird=eagle", eagleTim);
assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name=Tim", eagleTim, labradorTim);
assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim);
assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny);
assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny);
private Path dataDirectory;
private DataStore dataStore;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
dataStore = null;
tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null;
}
public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory);
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(partitionId, labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(partitionId, labradorTim));
assertSearch(dateRange, "bird=eagle", eagleTim);
assertSearch(dateRange, "dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name=Tim", eagleTim, labradorTim);
assertSearch(dateRange, "dog=labrador and name=Tim", labradorTim);
assertSearch(dateRange, "dog=labrador and !name=Tim", labradorJenny);
assertSearch(dateRange, "name=Jennifer or name=Jenny", pigeonJennifer, flamingoJennifer, labradorJenny);
// a͟n͟d binds stronger than o͟r
assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim);
assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim);
// a͟n͟d binds stronger than o͟r
assertSearch(dateRange, "name=Tim and dog=labrador or bird=pigeon", pigeonJennifer, labradorTim);
assertSearch(dateRange, "bird=pigeon or name=Tim and dog=labrador", pigeonJennifer, labradorTim);
// parenthesis override priority of a͟n͟d
assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim);
assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim);
// parenthesis override priority of a͟n͟d
assertSearch(dateRange, "name=Tim and (dog=labrador or bird=pigeon)", labradorTim);
assertSearch(dateRange, "(dog=labrador or bird=pigeon) and name=Tim", labradorTim);
// wildcards
assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim);
// wildcards
assertSearch(dateRange, "bird=*", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "name=Jen*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "dog=*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=lab*dor", labradorJenny, labradorTim);
assertSearch(dateRange, "dog=*lab*dor*", labradorJenny, labradorTim);
// 'in' queries
assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim);
assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim);
// 'in' queries
assertSearch(dateRange, "bird=(eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch(dateRange, "dog = (labrador) and name =Tim,Jennifer", labradorTim);
assertSearch(dateRange, "name =Jenn*", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch(dateRange, "name = (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch(dateRange, "name =XYZ, * and dog=labrador", labradorJenny, labradorTim);
}
}
public void testGetByTags() throws IOException {
public void testGetByTags() throws IOException {
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(partitionId, flamingoJennifer));
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
}
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(partitionId, flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
}
public void testBlockAlignment() throws IOException {
public void testBlockAlignment() throws IOException {
dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0);
}
dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assert.assertEquals(eagleTimBlockOffset % BSFile.BLOCK_SIZE, 0);
}
@DataProvider(name = "providerProposals")
public Iterator<Object[]> providerProposals() {
@DataProvider(name = "providerProposals")
public Iterator<Object[]> providerProposals() {
final List<Object[]> result = new ArrayList<>();
final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") });
result.add(new Object[] { "type=bird and subtype=eagle and name=|", "name", Arrays.asList("Tim") });
// returns Tim, because it is the only dog's name starting with 'Ti'
result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") });
// returns Tim, because it is the only dog's name starting with 'Ti'
result.add(new Object[] { "!name=Ti| and type=dog", "name", Arrays.asList("Tim") });
// all cats
result.add(new Object[] { "type=cat and !name=|", "name",
Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") });
// all cats
result.add(new Object[] { "type=cat and !name=|", "name",
Arrays.asList("Jane", "John", "Paul", "Sam", "Timothy") });
// finds nothing, because there are not dogs names neither Jenny, nor Ti*
result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() });
// finds nothing, because there are not dogs names neither Jenny, nor Ti*
result.add(new Object[] { "!name=Ti| and type=dog and !name=Jenny", "name", Arrays.asList() });
result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name",
Arrays.asList("Jenny", "Tim") });
result.add(new Object[] { "(type=bird and age=three or type=dog and age=three) and name=|", "name",
Arrays.asList("Jenny", "Tim") });
// all but Jennifer
result.add(new Object[] { "!(type=bird) and name=|", "name",
Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") });
// all but Jennifer
result.add(new Object[] { "!(type=bird) and name=|", "name",
Arrays.asList("Jane", "Jenny", "John", "Paul", "Sam", "Tim", "Timothy") });
result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") });
result.add(new Object[] { "type=bird and !subtype=eagle and name=|", "name", Arrays.asList("Jennifer") });
// DeMorgan
// TODO should only match "Jenny", because Jenny is the only non-bird name
// starting with 'Jen'
result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") });
// DeMorgan
// TODO should only match "Jenny", because Jenny is the only non-bird name
// starting with 'Jen'
result.add(new Object[] { "!(type=bird and name=Jen|)", "name", Arrays.asList("Jennifer", "Jenny") });
result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name",
Arrays.asList("Jennifer", "Jenny", "Tim") });
result.add(new Object[] { "!(type=dog and name=|) and !type=cat", "name",
Arrays.asList("Jennifer", "Jenny", "Tim") });
// not existing field
result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() });
// not existing field
result.add(new Object[] { "name=| and XYZ=Tim", "name", Arrays.asList() });
// not existing value
result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() });
// not existing value
result.add(new Object[] { "name=| and type=XYZ", "name", Arrays.asList() });
return result.iterator();
}
return result.iterator();
}
@Test(dataProvider = "providerProposals")
public void testProposals(final String queryWithCaret, final String field,
final List<String> expectedProposedValues) throws Exception {
@Test(dataProvider = "providerProposals")
public void testProposals(final String queryWithCaret, final String field,
final List<String> expectedProposedValues) throws Exception {
dataStore = new DataStore(dataDirectory);
final ParititionId partitionId = DateIndexExtension.now();
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
dataStore = new DataStore(dataDirectory);
final ParititionId partitionId = DateIndexExtension.now();
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
assertProposals(dateRange, queryWithCaret, field, expectedProposedValues);
}
assertProposals(dateRange, queryWithCaret, field, expectedProposedValues);
}
public void testIdenticalDatesGoIntoSameFile() throws Exception {
public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (final DataStore dataStore = new DataStore(dataDirectory)) {
try (final DataStore dataStore = new DataStore(dataDirectory)) {
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2);
dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2);
Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache");
}
}
Assert.assertEquals(dataStore.sizeWriterCache(), 1, "size of the writer cache");
}
}
public static void main(final String[] args) throws IOException, InterruptedException {
final Path dir = Files.createTempDirectory("pdb");
try (final DataStore dataStore = new DataStore(dir)) {
public static void main(final String[] args) throws IOException, InterruptedException {
final Path dir = Files.createTempDirectory("pdb");
try (final DataStore dataStore = new DataStore(dir)) {
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"),
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
final JFrame frame = new JFrame();
final JTextField input = new JTextField();
final JTextArea output = new JTextArea();
final JTextArea info = new JTextArea();
final JFrame frame = new JFrame();
final JTextField input = new JTextField();
final JTextArea output = new JTextArea();
final JTextArea info = new JTextArea();
frame.add(input, BorderLayout.NORTH);
frame.add(output, BorderLayout.CENTER);
frame.add(info, BorderLayout.SOUTH);
frame.add(input, BorderLayout.NORTH);
frame.add(output, BorderLayout.CENTER);
frame.add(info, BorderLayout.SOUTH);
input.setText("type=bird and !subtype=eagle and name=");
input.setText("type=bird and !subtype=eagle and name=");
input.addKeyListener(new KeyAdapter() {
input.addKeyListener(new KeyAdapter() {
@Override
public void keyReleased(final KeyEvent e) {
@Override
public void keyReleased(final KeyEvent e) {
final String query = input.getText();
final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT);
final String query = input.getText();
final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT);
final List<Proposal> proposals = dataStore.propose(q);
final List<Proposal> proposals = dataStore.propose(q);
final StringBuilder out = new StringBuilder();
final StringBuilder out = new StringBuilder();
for (final Proposal proposal : proposals) {
out.append(proposal.getProposedTag());
out.append(" ");
out.append(proposal.getProposedQuery());
out.append("\n");
}
for (final Proposal proposal : proposals) {
out.append(proposal.getProposedTag());
out.append(" ");
out.append(proposal.getProposedQuery());
out.append("\n");
}
final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString();
final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, "|").toString();
out.append("\n");
out.append("\n");
out.append("input: " + queryWithCaretMarker);
out.append("\n");
out.append("\n");
out.append("input: " + queryWithCaretMarker);
output.setText(out.toString());
output.setText(out.toString());
}
});
final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS)));
final StringBuilder out = new StringBuilder();
out.append("info\n");
for (final Doc doc : docs) {
out.append(doc.getTags());
out.append("\n");
}
info.setText(out.toString());
}
});
final List<Doc> docs = dataStore.search(Query.createQuery("", DateTimeRange.relative(1, ChronoUnit.DAYS)));
final StringBuilder out = new StringBuilder();
out.append("info\n");
for (final Doc doc : docs) {
out.append(doc.getTags());
out.append("\n");
}
info.setText(out.toString());
frame.setSize(800, 600);
frame.setVisible(true);
TimeUnit.HOURS.sleep(1000);
}
}
frame.setSize(800, 600);
frame.setVisible(true);
TimeUnit.HOURS.sleep(1000);
}
}
private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field,
final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));
proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery()));
final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag);
Collections.sort(proposedValues);
Collections.sort(expectedProposedValues);
Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:");
}
private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) {
final List<Doc> result = dataStore.search(new Query(query, dateRange));
Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't.");
}
private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) {
final Query query = new Query(queryString, dateRange);
final List<Doc> actualDocs = dataStore.search(query);
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual);
}
private void assertProposals(final DateTimeRange dateRange, final String queryWithCaret, final String field,
final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));
proposals.forEach(p -> assertQueryFindsResults(dateRange, p.getNewQuery()));
final List<String> proposedValues = CollectionUtils.map(proposals, Proposal::getProposedTag);
Collections.sort(proposedValues);
Collections.sort(expectedProposedValues);
Assert.assertEquals(proposedValues.toString(), expectedProposedValues.toString(), "proposed values:");
}
private void assertQueryFindsResults(final DateTimeRange dateRange, final String query) {
final List<Doc> result = dataStore.search(new Query(query, dateRange));
Assert.assertFalse(result.isEmpty(), "The query '" + query + "' must return a result, but didn't.");
}
private void assertSearch(final DateTimeRange dateRange, final String queryString, final Tags... tags) {
final Query query = new Query(queryString, dateRange);
final List<Doc> actualDocs = dataStore.search(query);
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + queryString + " Found: " + actual);
}
}

198
data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java
View File

@@ -16,129 +16,129 @@ import org.testng.annotations.Test;
@Test
public class DateIndexExtensionTest {
@DataProvider
public Object[][] provider() {
@DataProvider
public Object[][] provider() {
final List<Object[]> result = new ArrayList<>();
final List<Object[]> result = new ArrayList<>();
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected });
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected });
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected });
}
return result.toArray(new Object[0][]);
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
@Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
final DateTimeRange dateRange = new DateTimeRange(start, end);
final DateTimeRange dateRange = new DateTimeRange(start, end);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
Assert.assertEquals(actual, expected);
}
Assert.assertEquals(actual, expected);
}
public void testDateToDateIndexPrefix() {
public void testDateToDateIndexPrefix() {
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
}
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
}
public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12));
public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12));
final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802);
final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801);
final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712);
final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802);
final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801);
final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712);
final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension
.toPartitionIds(range_201712_201802);
Assert.assertEquals(dateIndexPrefixesWithEmptyCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")));
final List<ParititionId> dateIndexPrefixesWithEmptyCache = DateIndexExtension
.toPartitionIds(range_201712_201802);
Assert.assertEquals(dateIndexPrefixesWithEmptyCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")));
final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension
.toPartitionIds(range_201712_201801);
Assert.assertEquals(dateIndexPrefixesWithFilledCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801")));
final List<ParititionId> dateIndexPrefixesWithFilledCache = DateIndexExtension
.toPartitionIds(range_201712_201801);
Assert.assertEquals(dateIndexPrefixesWithFilledCache,
Arrays.asList(new ParititionId("201712"), new ParititionId("201801")));
final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712);
Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712")));
}
final List<ParititionId> dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712);
Assert.assertEquals(dateIndexPrefixesOneMonth, Arrays.asList(new ParititionId("201712")));
}
public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final List<Long> dateIndexEpochMillis = DateIndexExtension
.toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802));
Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802));
}
final List<Long> dateIndexEpochMillis = DateIndexExtension
.toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802));
Assert.assertEquals(dateIndexEpochMillis, Arrays.asList(exp_201712, exp_201801, exp_201802));
}
public void testPerformance() {
public void testPerformance() {
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final int iterations = 1_000_000;
final int factor = 1;
final int warmup = 20 * factor;
final int rounds = warmup + 20;
final int iterations = 1_000_000;
final int factor = 1;
final int warmup = 20 * factor;
final int rounds = warmup + 20;
// fill the cache
DateIndexExtension.DATE_PREFIX_CACHE.clear();
for (long i = min; i < max; i += 3600 * 24 * 28) {
DateIndexExtension.toPartitionId(i);
}
// fill the cache
DateIndexExtension.DATE_PREFIX_CACHE.clear();
for (long i = min; i < max; i += 3600 * 24 * 28) {
DateIndexExtension.toPartitionId(i);
}
final List<Double> measurements = new ArrayList<>();
final List<Double> measurements = new ArrayList<>();
for (int r = 0; r < rounds; r++) {
for (int r = 0; r < rounds; r++) {
final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) {
DateIndexExtension.toPartitionId(mid);
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms");
measurements.add(duration);
}
final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) {
DateIndexExtension.toPartitionId(mid);
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms");
measurements.add(duration);
}
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics();
System.out.println(stats);
}
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics();
System.out.println(stats);
}
}

465
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -22,275 +22,276 @@ import org.testng.annotations.Test;
@Test
public class ProposerTest {
private Path dataDirectory;
private DataStore dataStore;
private DateTimeRange dateRange;
private Path dataDirectory;
private DataStore dataStore;
private DateTimeRange dateRange;
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
dataStore.close();
dataStore = null;
Tags.STRING_COMPRESSOR = null;
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
dataStore.close();
dataStore = null;
Tags.STRING_COMPRESSOR = null;
}
private void initDatabase() throws Exception {
dataStore = new DataStore(dataDirectory);
dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
private void initDatabase() throws Exception {
dataStore = new DataStore(dataDirectory);
dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD);
}
dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD);
}
public void testEmptyQuery() throws Exception {
public void testEmptyQuery() throws Exception {
assertProposals("|", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
assertProposals("|", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
assertProposals(" |", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
}
assertProposals(" |", ResultMode.FULL_VALUES, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
);
assertProposals("|bird", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird|", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird=eagle and n|", ResultMode.FULL_VALUES, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
);
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim|", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("|bird", ResultMode.FULL_VALUES, //
new Proposal("bird", "bird=* ", true, "bird=", 5), //
new Proposal("dog", "dog=* ", true, "dog=", 4), //
new Proposal("method", "method=* ", true, "method=", 7), //
new Proposal("name", "name=* ", true, "name=", 5), //
new Proposal("source", "source=* ", true, "source=", 7) //
);
}
assertProposals("name =Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
// TODO this case is currently handled completely wrong - it is handled similar to an empty query
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim|", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("name =Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
assertProposals("name =Tim,Je|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
// TODO this case is currently handled completely wrong - it is handled similar
// to an empty query
// assertProposals("|bird=eagle and name=Tim", ResultMode.FULL_VALUES, //
// new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
// new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
// );
/*
*/
}
@Test(enabled = true)
public void testInExpressions() throws Exception {
assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
/*
*/
}
assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
@Test(enabled = true)
public void testInExpressions() throws Exception {
assertProposals("name = (Timothy,|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
assertProposals("name = (Timothy, J|)", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
/*
*/
}
assertProposals("name = (Tim|)", ResultMode.FULL_VALUES, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
public void testProposalOnEmptyValuePrefix() throws Exception {
assertProposals("name=|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
/*
*/
}
assertProposals("method=|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
);
assertProposals("method=|", ResultMode.FULL_VALUES, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
public void testProposalOnEmptyValuePrefix() throws Exception {
assertProposals("name=|", ResultMode.FULL_VALUES, //
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
public void testProposalOnValueSmartExpression() throws Exception {
assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
);
assertProposals("method=|", ResultMode.FULL_VALUES, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
public void testProposalOnValueSmartExpression() throws Exception {
assertProposals("method=Foo.|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
// returns nothing, because GloblikePattern.globlikeToRegex() returns the
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
// Maybe I will change that some day and allow upper case characters before
// "Stuff".
assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT);
assertProposals("method=Foo.*Stuf|", ResultMode.CUT_AT_DOT, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34) //
);
}
// returns nothing, because GloblikePattern.globlikeToRegex() returns the
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
// Maybe I will change that some day and allow upper case characters before
// "Stuff".
assertProposals("method=Foo.Stuf|", ResultMode.CUT_AT_DOT);
public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and |", ResultMode.FULL_VALUES, //
proposal("name", "name=* and name=* ", "name=* and name=|"), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), //
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34) //
);
}
public void testProposalWithWildcards() throws Exception {
public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and |", ResultMode.FULL_VALUES, //
proposal("name", "name=* and name=* ", "name=* and name=|"), //
proposal("bird", "name=* and bird=* ", "name=* and bird=|"), //
proposal("dog", "name=* and dog=* ", "name=* and dog=|"), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
proposal("method", "name=* and method=* ", "name=* and method=|"), //
proposal("source", "name=* and source=* ", "name=* and source=|")//
);
}
assertProposals("name=*im|", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim", "name=Tim|"), //
proposal("Timothy", "name=Timothy", "name=Timothy|")//
);
public void testProposalWithWildcards() throws Exception {
assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, //
proposal("FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), //
proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
assertProposals("name=*im|", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim", "name=Tim|"), //
proposal("Timothy", "name=Timothy", "name=Timothy|")//
);
public void testProposalWithAndExpression() throws Exception {
assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), //
proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")//
);
assertProposals("(method=FooService.doIS,FooController.*) and method=|", ResultMode.FULL_VALUES, //
proposal("FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooService.doImportantStuff|"), //
proposal("FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff",
"(method=FooService.doIS,FooController.*) and method=FooController.doImportantStuff|")//
);
}
assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), //
proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")//
);
}
public void testProposalWithAndExpression() throws Exception {
assertProposals("name=*im| and bird=eagle", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle", "name=Tim| and bird=eagle"), //
proposal("Timothy", "name=Timothy and bird=eagle", "name=Timothy| and bird=eagle")//
);
public void testProposalWithAndNotExpression() throws Exception {
assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle",
"name=Tim and ! dog=labrador and bird=eagle|") //
);
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
assertProposals("name=*im| and bird=eagle,pigeon", ResultMode.FULL_VALUES, //
proposal("Tim", "name=Tim and bird=eagle,pigeon", "name=Tim| and bird=eagle,pigeon"), //
proposal("Timothy", "name=Timothy and bird=eagle,pigeon", "name=Timothy| and bird=eagle,pigeon")//
);
}
private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) {
final String newQueryWithoutCaretMarker = newQuery.replace("|", "");
final int newCaretPosition = newQuery.indexOf('|');
return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition);
}
public void testProposalWithAndNotExpression() throws Exception {
assertProposals("name=Tim and ! dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and ! dog=labrador and bird=eagle",
"name=Tim and ! dog=labrador and bird=eagle|") //
);
assertProposals("name=Tim and not dog=labrador and bird=|", ResultMode.FULL_VALUES, //
proposal("eagle", "name=Tim and not dog=labrador and bird=eagle",
"name=Tim and not dog=labrador and bird=eagle|") //
);
}
private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected)
throws InterruptedException {
final int caretIndex = query.indexOf("|");
final String q = query.replace("|", "");
assertProposals(q, resultMode, caretIndex, expected);
}
private Proposal proposal(final String proposedTag, final String proposedQuery, final String newQuery) {
final String newQueryWithoutCaretMarker = newQuery.replace("|", "");
final int newCaretPosition = newQuery.indexOf('|');
return new Proposal(proposedTag, proposedQuery, true, newQueryWithoutCaretMarker, newCaretPosition);
}
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
final Proposal... expected) throws InterruptedException {
private void assertProposals(final String query, final ResultMode resultMode, final Proposal... expected)
throws InterruptedException {
final int caretIndex = query.indexOf("|");
final String q = query.replace("|", "");
assertProposals(q, resultMode, caretIndex, expected);
}
final List<Proposal> actual = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
final Proposal... expected) throws InterruptedException {
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString)));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString)));
Assert.assertEquals(actual, expectedList);
}
final List<Proposal> actual = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + String.join("\n", CollectionUtils.map(actual, Proposal::toString)));
System.out.println("expected: " + String.join("\n", CollectionUtils.map(expectedList, Proposal::toString)));
Assert.assertEquals(actual, expectedList);
}
}

78
data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java
View File

@@ -21,53 +21,53 @@ import org.testng.annotations.Test;
@Test
public class QueryCompletionIndexTest {
private Path dataDirectory;
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void test() throws Exception {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
public void test() throws Exception {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
);
final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
);
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) {
for (final Tags t : tags) {
index.addTags(partitionId, t);
}
try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) {
for (final Tags t : tags) {
index.addTags(partitionId, t);
}
// all firstnames where lastname=Doe are returned sorted alphabetically.
// tags A and B match
final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"),
"firstname");
Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John"));
// all firstnames where lastname=Doe are returned sorted alphabetically.
// tags A and B match
final SortedSet<String> firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"),
"firstname");
Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John"));
// no duplicates are returned:
// tags A and C match firstname=John, but both have country=Atlantis
final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"),
"country");
Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis"));
// no duplicates are returned:
// tags A and C match firstname=John, but both have country=Atlantis
final SortedSet<String> countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"),
"country");
Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis"));
// findAllValuesForField sorts alphabetically
final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname");
Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames);
// findAllValuesForField sorts alphabetically
final SortedSet<String> firstnames = index.findAllValuesForField(dateRange, "firstname");
Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames);
final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country");
Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado"));
}
}
final SortedSet<String> countries = index.findAllValuesForField(dateRange, "country");
Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado"));
}
}
}

88
data-store/src/test/java/org/lucares/pdb/datastore/lang/CandidateGrouperTest.java
View File

@@ -12,54 +12,54 @@ import org.testng.annotations.Test;
@Test
public class CandidateGrouperTest {
@DataProvider
public Object[][] providerGroup() {
final List<Object[]> result = new ArrayList<>();
@DataProvider
public Object[][] providerGroup() {
final List<Object[]> result = new ArrayList<>();
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = |", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = a|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.|", //
Set.of("aa.xx.", "aa.yy.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.x|", //
Set.of("aa.xx.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.xx.|", //
Set.of("aa.xx.AA.", "aa.xx.BB") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), //
"name = aa.xx.AA.|", //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") });
result.add(new Object[] { //
Set.of("XX.YY.ZZ", "XX.YY"), //
"name = XX.Y|", //
Set.of("XX.YY.", "XX.YY") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = |", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = a|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa|", //
Set.of("aa.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.yy.BB", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.|", //
Set.of("aa.xx.", "aa.yy.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.x|", //
Set.of("aa.xx.") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.BB", "aa.xx.AA.YY"), //
"name = aa.xx.|", //
Set.of("aa.xx.AA.", "aa.xx.BB") });
result.add(new Object[] { //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY"), //
"name = aa.xx.AA.|", //
Set.of("aa.xx.AA.XX", "aa.xx.AA.YY") });
result.add(new Object[] { //
Set.of("XX.YY.ZZ", "XX.YY"), //
"name = XX.Y|", //
Set.of("XX.YY.", "XX.YY") });
return result.toArray(new Object[0][]);
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "providerGroup")
public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) {
final CandidateGrouper grouper = new CandidateGrouper();
@Test(dataProvider = "providerGroup")
public void testGroup(final Set<String> values, final String queryWithCaretMarker, final Set<String> expected) {
final CandidateGrouper grouper = new CandidateGrouper();
final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER);
final String query = queryWithCaretMarker.replace("|", NewProposerParser.CARET_MARKER);
final SortedSet<String> actual = grouper.group(values, query);
final SortedSet<String> actual = grouper.group(values, query);
Assert.assertEquals(actual, expected);
}
Assert.assertEquals(actual, expected);
}
}