andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

replace the FolderStorage with DiskStorage

Browse Source 
- The DiskStorage uses only one file instead of millions.
  Also the block size is only 512 byte instead of 4kb, which
  helps to reduce the memory usage for short sequences.
- Update primitiveCollections to get the new LongList.range
  and LongList.rangeClosed methods.
- BSFile now stores Time&Value sequences and knows how to
  encode the time values with delta encoding.
- Doc had to do some magic tricks to save memory. The path
  was initialized lazy and stored as byte array. This is no
  longer necessary. The patch was replaced by the
  rootBlockNumber of the BSFile.
- Had to temporarily disable the 'in' queries.
- The stored values are now processed as stream of LongLists
  instead of Entry. The overhead for creating Entries is
  gone, so is the memory overhead, because Entry was an
  object and had a reference to the tags, which is
  unnecessary.
This commit is contained in:
Andreas Huber
2018-09-12 09:35:07 +02:00
parent 26dc052b95
commit 1182d76205
36 changed files with 799 additions and 1483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
data-store/build.gradle
View File

@@ -4,9 +4,10 @@ dependencies {
compile project(':pdb-api')
compile project(':file-utils')
compile project(':pdb-utils')
compile project(':block-storage')
antlr "org.antlr:antlr4:4.7.1"
compile 'org.lucares:primitiveCollections:0.1.20180817193843'
compile 'org.lucares:primitiveCollections:0.1.20180908084945'
compile 'org.apache.commons:commons-lang3:3.7'
compile 'com.google.guava:guava:26.0-jre'

5
data-store/src/main/antlr/org/lucares/pdb/datastore/lang/PdbLang.g4
View File

@@ -10,7 +10,8 @@ expression
: LPAREN expression RPAREN #parenExpression
| NOT expression #notExpression
| prop=identifier eq=equal value=propValue #propertyExpression
| prop=identifier in=inExpr LPAREN listOfProperties=listOfPropValues RPAREN #inExpression
//| prop=identifier in=inExpr LPAREN listOfProperties=listOfPropValues RPAREN #inExpression
| '_in' prop=identifier in=inExpr LPAREN listOfProperties=listOfPropValues RPAREN #inExpression
| left=expression AND right=expression #binaryAndExpression
| left=expression OR right=expression #binaryOrExpression
;
@@ -37,7 +38,7 @@ EQUAL : '=' ;
IN : 'in' ;
LPAREN : '(' ;
RPAREN : ')' ;
COMMA : ',' ;
COMMA : ',' ;
IDENTIFIER
: JavaLetter JavaLetterOrDigit*
;

68
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -1,17 +1,15 @@
package org.lucares.pdb.datastore;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.blockstorage.BSFile;
public class Doc {
private final Tags tags;
private final long offsetInListingFile;
private final Path storageBasePath;
private byte[] path;
/**
* the block number used by {@link BSFile}
*/
private final long rootBlockNumber;
/**
* Initializes a new document.
@@ -23,64 +21,32 @@ public class Doc {
* This is used to reduce the memory footprint.
*
* @param tags
* @param offsetInListingFile
* must be set if {@code path} is {@code null}
* @param storageBasePath
* the storage base path.
* @param relativePath
* optional, can be {@code null}. This path is relative to
* {@code storageBasePath}
* @param offsetInListingFile must be set if {@code path} is {@code null}
* @param storageBasePath the storage base path.
* @param relativePath optional, can be {@code null}. This path is
* relative to {@code storageBasePath}
*/
public Doc(final Tags tags, final long offsetInListingFile, final Path storageBasePath, final Path relativePath) {
super();
public Doc(final Tags tags, final long rootBlockNumber) {
this.tags = tags;
this.offsetInListingFile = offsetInListingFile;
this.storageBasePath = storageBasePath;
setRelativePath(relativePath);
this.rootBlockNumber = rootBlockNumber;
}
public Tags getTags() {
return tags;
}
public void setRelativePath(final Path path) {
if (path != null) {
this.path = path.toString().getBytes(StandardCharsets.UTF_8);
} else {
this.path = null;
}
}
/**
* The path to the storage file.
* <p>
* This value is lazily initialized. Callers have to provide a resolver. See
* {@link DataStore#getFolderStoragePathResolver()}.
* the block number used by {@link BSFile}
*
* @return the path
* @return the root block number of this document
*/
public Path getAbsolutePath(final FolderStoragePathResolver resolver) {
if (path == null) {
final Path resolvedPath = resolver.getPath(offsetInListingFile);
setRelativePath(resolvedPath);
}
final Path relativePath = Paths.get(new String(path, StandardCharsets.UTF_8));
return storageBasePath.resolve(relativePath);
}
private Path getAbsolutePathNullable() {
return getAbsolutePath(FolderStoragePathResolver.NULL);
}
public long getOffsetInListingFile() {
return offsetInListingFile;
public long getRootBlockNumber() {
return rootBlockNumber;
}
@Override
public String toString() {
return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getAbsolutePathNullable()
+ "]";
return "Doc [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
}

9
data-store/src/main/java/org/lucares/pdb/datastore/FolderStoragePathResolver.java
View File

@@ -1,9 +0,0 @@
package org.lucares.pdb.datastore;
import java.nio.file.Path;
public interface FolderStoragePathResolver {
FolderStoragePathResolver NULL = offset -> null;
public Path getPath(long offsetInListingFile);
}

11
data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java
View File

@@ -8,6 +8,7 @@ import java.util.SortedSet;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.datastore.internal.Proposer;
import org.lucares.pdb.diskstorage.DiskStorage;
public class PdbDB implements AutoCloseable {
@@ -23,7 +24,7 @@ public class PdbDB implements AutoCloseable {
return dataStore.search(query);
}
public Path createNewFile(final Tags tags) throws IOException {
public long createNewFile(final Tags tags) throws IOException {
return dataStore.createNewFile(tags);
}
@@ -44,10 +45,6 @@ public class PdbDB implements AutoCloseable {
return dataStore.getByTags(tags);
}
public FolderStoragePathResolver getFolderStoragePathResolver() {
return dataStore.getFolderStoragePathResolver();
}
public Path getStorageBasePath() {
return dataStore.getStorageBasePath();
}
@@ -56,4 +53,8 @@ public class PdbDB implements AutoCloseable {
public void close() throws IOException {
dataStore.close();
}
public DiskStorage getDiskStorage() {
return dataStore.getDiskStorage();
}
}

103
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -1,6 +1,9 @@
package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
@@ -9,19 +12,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.collections.IntList;
import org.lucares.pdb.api.StringCompressor;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.FolderStoragePathResolver;
import org.lucares.pdb.datastore.lang.Expression;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds;
import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.lucares.pdb.diskstorage.DiskStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,8 +37,12 @@ public class DataStore implements AutoCloseable {
private static final Logger INITIALIZE = LoggerFactory.getLogger("org.lucares.metrics.dataStore.init");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.US_ASCII);
public static final char LISTING_FILE_SEPARATOR = ',';
private static final byte[] LISTING_FILE_SEPARATOR_BYTES = String.valueOf(LISTING_FILE_SEPARATOR)
.getBytes(StandardCharsets.US_ASCII);
private static final String SUBDIR_STORAGE = "storage";
private static final String PDB_EXTENSION = ".pdb";
// to be guarded by itself
private final List<Doc> docIdToDoc = new ArrayList<>();
@@ -41,36 +51,36 @@ public class DataStore implements AutoCloseable {
private final ConcurrentHashMap<String, Map<String, IntList>> keyToValueToDocId = new ConcurrentHashMap<>();
private final FolderStorage folderStorage;
private final FolderStoragePathResolver folderStoragePathResolver;
private final DiskStorage diskStorage;
private final Path diskStorageFilePath;
private final Path storageBasePath;
private final Path listingFilePath;
private final RandomAccessFile listingFile;
public DataStore(final Path dataDirectory) throws IOException {
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(dataDirectory));
storageBasePath = storageDirectory(dataDirectory);
folderStorage = new FolderStorage(storageBasePath, 1000);
init(folderStorage);
listingFilePath = storageBasePath.resolve("listing.csv");
diskStorageFilePath = storageBasePath.resolve("data.bs");
diskStorage = new DiskStorage(diskStorageFilePath);
initListingFileIfNotExists();
init(diskStorage);
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rw");
folderStoragePathResolver = folderStorage::getPathByOffset;
}
private void init(final FolderStorage folderStorage) throws IOException {
private void init(final DiskStorage diskStorage) throws IOException {
final long start = System.nanoTime();
final Stream<ListingFileEntry> files = folderStorage.list();
files.parallel()
.forEach(listingFileEntry -> {
final Stream<ListingFileEntry> files = list();
files.parallel().forEach(listingFileEntry -> {
listingFileEntry.unsetRelativePath(); // unset the path, so that we don't store it for every document (will
// be
// initialized lazily if needed)
final String filename = listingFileEntry.getSerializedTags();
final Tags tags = toTags(filename);
cacheTagToFileMapping(tags, listingFileEntry);
final String filename = listingFileEntry.getFilename();
final Tags tags = toTags(filename);
cacheTagToFileMapping(tags, listingFileEntry);
});
});
trimIntLists();
sortIntLists();
synchronized (docIdToDoc) {
@@ -79,11 +89,19 @@ public class DataStore implements AutoCloseable {
INITIALIZE.info(((System.nanoTime() - start) / 1_000_000.0) + "ms");
}
public Stream<ListingFileEntry> list() throws IOException {
final ListingFileIterator iterator = new ListingFileIterator(listingFilePath);
final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
}
private void cacheTagToFileMapping(final Tags tags, final ListingFileEntry listingFileEntry) {
final int docId;
final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), storageBasePath,
listingFileEntry.getPath());
final Doc newDoc = new Doc(tags, listingFileEntry.getRootBlockNumber());
synchronized (docIdToDoc) {
docId = docIdToDoc.size();
docIdToDoc.add(newDoc);
@@ -149,14 +167,16 @@ public class DataStore implements AutoCloseable {
return dataDirectory.resolve(SUBDIR_STORAGE);
}
public Path createNewFile(final Tags tags) throws IOException {
public long createNewFile(final Tags tags) throws IOException {
final String filename = tags.getFilename();
final ListingFileEntry listingFileEntry = folderStorage.insert(filename, PDB_EXTENSION);
final String filename = tags.serialize();
final long newFilesRootBlockNumber = diskStorage.appendNewBlock();
updateListingFile(tags, newFilesRootBlockNumber);
final ListingFileEntry listingFileEntry = new ListingFileEntry(filename, newFilesRootBlockNumber);
cacheTagToFileMapping(tags, listingFileEntry);
return listingFileEntry.getPath();
return newFilesRootBlockNumber;
}
private Tags toTags(final String filename) {
@@ -245,16 +265,39 @@ public class DataStore implements AutoCloseable {
return result;
}
public FolderStoragePathResolver getFolderStoragePathResolver() {
return folderStoragePathResolver;
}
public Path getStorageBasePath() {
return storageBasePath;
}
@Override
public void close() throws IOException {
folderStorage.close();
diskStorage.close();
}
private void initListingFileIfNotExists() throws IOException {
if (!Files.exists(listingFilePath)) {
LOGGER.info("listing file not found -> creating a new one");
Files.createFile(listingFilePath);
}
}
private synchronized ListingFileEntry updateListingFile(final Tags tags, final long newFilesRootBlockNumber)
throws IOException {
final long offsetInListingFile = Files.size(listingFilePath);
// remember: all paths within storageBaseDirectory use only ascii characters
listingFile.seek(offsetInListingFile);
listingFile.write(tags.serialize().getBytes(StandardCharsets.US_ASCII));
listingFile.write(LISTING_FILE_SEPARATOR_BYTES);
listingFile.write(Long.toString(newFilesRootBlockNumber).getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE);
return new ListingFileEntry(tags.serialize(), newFilesRootBlockNumber);
}
public DiskStorage getDiskStorage() {
return diskStorage;
}
}

187
data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java
View File

@@ -1,187 +0,0 @@
package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Iterator;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.pdb.api.RuntimeIOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FolderStorage implements AutoCloseable {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.US_ASCII);
static final String LISTING_FILE_NAME = "listing.csv";
private final static Logger LOGGER = LoggerFactory.getLogger(FolderStorage.class);
private final static Logger METRICS_CREATE_LISTING_FILE = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.createListingFile");
private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.getPathByOffset");
private final static Logger METRICS_INSERT = LoggerFactory.getLogger("org.lucares.metrics.folderStorage.insert");
private final Path storageBaseDirectory;
private int firstLevel = 0;
private int secondLevel = 0;
private int filesInSecondLevel = 0;
private Path currentDirectory;
private final int maxFilesPerFolder;
private final Path listingFilePath;
private final RandomAccessFile listingFile;
public FolderStorage(final Path storageBaseDirectory, final int maxFilesPerFolder) throws IOException {
this.storageBaseDirectory = storageBaseDirectory;
this.listingFilePath = storageBaseDirectory.resolve(LISTING_FILE_NAME);
this.maxFilesPerFolder = maxFilesPerFolder;
init();
initListingFileIfNotExists();
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rw");
}
@Override
public void close() throws IOException {
listingFile.close();
}
private void init() throws IOException {
Files.createDirectories(storageBaseDirectory);
firstLevel = Math.max((int) Files.list(storageBaseDirectory).filter(Files::isDirectory).count() - 1, 0);
final Path firstLevelDirectory = storageBaseDirectory.resolve(String.valueOf(firstLevel));
Files.createDirectories(firstLevelDirectory);
secondLevel = Math.max((int) Files.list(firstLevelDirectory).filter(Files::isDirectory).count() - 1, 0);
currentDirectory = firstLevelDirectory.resolve(String.valueOf(secondLevel));
Files.createDirectories(currentDirectory);
filesInSecondLevel = (int) Files.list(currentDirectory).count();
}
public ListingFileEntry insert(final String filenamePrefix, final String filenameSuffix) throws IOException {
final long start = System.nanoTime();
ensureCapacity();
String filename = filenamePrefix + "$" + filenameSuffix;
int index = 1;
Path newFile = currentDirectory.resolve(filename);
while (Files.exists(newFile)) {
filename = filenamePrefix + "$" + index++ + filenameSuffix;
newFile = currentDirectory.resolve(filename);
}
Files.createFile(newFile);
filesInSecondLevel++;
final ListingFileEntry result = updateListingFile(newFile);
METRICS_INSERT.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
return result;
}
private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException {
final long offsetInListingFile = Files.size(listingFilePath);
// remember: all paths within storageBaseDirectory use only ascii characters
final Path relativePath = storageBaseDirectory.relativize(newFile);
listingFile.seek(offsetInListingFile);
listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE);
final String filename = newFile.getFileName().toString();
return new ListingFileEntry(filename, offsetInListingFile, relativePath);
}
private void ensureCapacity() throws IOException {
if (filesInSecondLevel >= maxFilesPerFolder) {
secondLevel++;
if (secondLevel >= maxFilesPerFolder) {
firstLevel++;
secondLevel = 0;
}
filesInSecondLevel = 0;
updateCurrentDirectory();
}
}
private void updateCurrentDirectory() throws IOException {
currentDirectory = storageBaseDirectory.resolve(String.valueOf(firstLevel))
.resolve(String.valueOf(secondLevel));
Files.createDirectories(currentDirectory);
}
public Stream<ListingFileEntry> list() throws IOException {
final ListingFileIterator iterator = new ListingFileIterator(listingFilePath);
final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
}
private void initListingFileIfNotExists() throws IOException {
if (!Files.exists(listingFilePath)) {
final long start = System.nanoTime();
LOGGER.info("listing file not found -> creating a new one");
createNewListingFile();
METRICS_CREATE_LISTING_FILE.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
}
}
private void createNewListingFile() throws IOException {
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matchRegularFiles = (path, attr) -> Files.isRegularFile(path);
// remember: all paths within storageBaseDirectory use only ascii characters
try (final Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII,
StandardOpenOption.CREATE, StandardOpenOption.APPEND);
final Stream<Path> stream = Files.find(storageBaseDirectory, maxDepth, matchRegularFiles)) {
final Iterator<Path> iterator = stream.iterator();
while (iterator.hasNext()) {
final Path path = iterator.next();
if (!path.getFileName().toString().equals(LISTING_FILE_NAME)) {
final Path relativePath = storageBaseDirectory.relativize(path);
out.write(relativePath.toString());
out.write("\n");
}
}
}
}
public synchronized Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException {
final long start = System.nanoTime();
try {
listingFile.seek(offsetInListingFile);
// remember: all paths within storageBaseDirectory use only ascii characters
final String line = listingFile.readLine();
return Paths.get(line);
} catch (final IOException e) {
throw new RuntimeIOException(e);
} finally {
METRICS_GET_PATH_BY_OFFSET.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
}
}
}

61
data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java
View File

@@ -1,15 +1,10 @@
package org.lucares.pdb.datastore.internal;
import java.nio.file.Path;
import javax.annotation.Nullable;
import org.lucares.pdb.datastore.Doc;
public class ListingFileEntry {
private final String filename;
private final long offsetInListingFile;
private Path relativePath;
private final String serializedTags;
private final long rootBlockNumber;
/**
* Create a new {@link ListingFileEntry}.
@@ -18,47 +13,33 @@ public class ListingFileEntry {
* the listing file, then the {@code path} is set to {@code null}. This is done
* to save memory. See {@link Doc} for more information on its usage.
*
* @param filename
* @param offsetInListingFile
* @param relativePath
* optional, see {@link Doc}
* @param serializedTags
* @param rootBlockNumber
*/
public ListingFileEntry(final String filename, final long offsetInListingFile, final Path relativePath) {
this.filename = filename;
this.offsetInListingFile = offsetInListingFile;
this.relativePath = relativePath;
public ListingFileEntry(final String serializedTags, final long rootBlockNumber) {
this.serializedTags = serializedTags;
this.rootBlockNumber = rootBlockNumber;
}
public String getFilename() {
return filename;
public String getSerializedTags() {
return serializedTags;
}
public long getOffsetInListingFile() {
return offsetInListingFile;
}
public void unsetRelativePath() {
relativePath = null;
}
@Nullable
public Path getPath() {
return relativePath;
public long getRootBlockNumber() {
return rootBlockNumber;
}
@Override
public String toString() {
return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile
+ ", relativePath=" + relativePath + "]";
return "ListingFileEntry [serializedTags=" + serializedTags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((filename == null) ? 0 : filename.hashCode());
result = prime * result + (int) (offsetInListingFile ^ (offsetInListingFile >>> 32));
result = prime * result + ((relativePath == null) ? 0 : relativePath.hashCode());
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((serializedTags == null) ? 0 : serializedTags.hashCode());
return result;
}
@@ -71,18 +52,14 @@ public class ListingFileEntry {
if (getClass() != obj.getClass())
return false;
final ListingFileEntry other = (ListingFileEntry) obj;
if (filename == null) {
if (other.filename != null)
return false;
} else if (!filename.equals(other.filename))
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (offsetInListingFile != other.offsetInListingFile)
return false;
if (relativePath == null) {
if (other.relativePath != null)
if (serializedTags == null) {
if (other.serializedTags != null)
return false;
} else if (!relativePath.equals(other.relativePath))
} else if (!serializedTags.equals(other.serializedTags))
return false;
return true;
}
}

29
data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileIterator.java
View File

@@ -1,7 +1,6 @@
package org.lucares.pdb.datastore.internal;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -46,25 +45,35 @@ public class ListingFileIterator implements Iterator<ListingFileEntry>, AutoClos
}
public ListingFileEntry getNext() {
final StringBuilder line = new StringBuilder();
final StringBuilder serializedTags = new StringBuilder();
final StringBuilder serializedRootBlockNumber = new StringBuilder();
try {
final long offsetInListingFile = is.getCount();
int state = 0; // 0 = reading serialized tags; 1 = reading root block number
int codePoint;
while ((codePoint = is.read()) >= 0) {
if (codePoint == '\n') {
break;
if (state == 0) {
if (codePoint == DataStore.LISTING_FILE_SEPARATOR) {
state = 1;
continue;
}
serializedTags.appendCodePoint(codePoint);
} else {
if (codePoint == '\n') {
break;
}
serializedRootBlockNumber.appendCodePoint(codePoint);
}
line.appendCodePoint(codePoint);
}
if (codePoint < 0) {
return null;
}
final int lastSeparatorPosition = line.lastIndexOf(File.separator);
final String filename = line.substring(lastSeparatorPosition + 1);
return new ListingFileEntry(filename, offsetInListingFile, null);
final String filename = serializedTags.toString();
final long rootBlockNumebr = Long.parseLong(serializedRootBlockNumber.toString());
return new ListingFileEntry(filename, rootBlockNumebr);
} catch (final IOException e) {
throw new RuntimeIOException(e);

2
data-store/src/main/java/org/lucares/pdb/datastore/lang/ProposerParser.java
View File

@@ -26,7 +26,7 @@ public class ProposerParser {
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final QueryCompletionPdbLangParser parser = new QueryCompletionPdbLangParser(tokens);
parser.setTrace(false);
parser.setTrace(true);
final Listener listener = parser.new Listener(query, dataStore, caretIndex);
parser.addErrorListener(listener);

117
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -3,12 +3,11 @@ package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Doc;
@@ -23,7 +22,7 @@ import org.testng.annotations.Test;
public class DataStoreTest {
private Path dataDirectory;
private DataStore dataStore;
private Map<Tags, Path> tagsToPath;
private Map<Tags, Long> tagsToBlockStorageRootBlockNumber;
@BeforeMethod
public void beforeMethod() throws IOException {
@@ -34,48 +33,26 @@ public class DataStoreTest {
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
dataStore = null;
tagsToPath = null;
tagsToBlockStorageRootBlockNumber = null;
Tags.STRING_COMPRESSOR = null;
}
public void testInsertSingleTag() throws Exception {
final Path path;
{
final DataStore dataStore = new DataStore(dataDirectory);
final Path storageBasePath = dataStore.getStorageBasePath();
final Tags tags = Tags.create("key1", "value1", "key2", "value2");
path = dataStore.createNewFile(tags);
assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
}
{
final DataStore dataStore = new DataStore(dataDirectory);
final Path storageBasePath = dataStore.getStorageBasePath();
assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
}
}
public void testQuery() throws Exception {
dataStore = new DataStore(dataDirectory);
tagsToPath = new LinkedHashMap<>();
final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim");
tagsToPath.put(eagleTim, null);
tagsToPath.put(pigeonJennifer, null);
tagsToPath.put(flamingoJennifer, null);
tagsToPath.put(labradorJenny, null);
tagsToPath.put(labradorTim, null);
for (final Tags tags : tagsToPath.keySet()) {
final Path newFile = dataStore.createNewFile(tags);
tagsToPath.put(tags, newFile);
}
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(eagleTim));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(flamingoJennifer));
tagsToBlockStorageRootBlockNumber.put(labradorJenny, dataStore.createNewFile(labradorJenny));
tagsToBlockStorageRootBlockNumber.put(labradorTim, dataStore.createNewFile(labradorTim));
assertSearch("bird=eagle", eagleTim);
assertSearch("dog=labrador", labradorJenny, labradorTim);
@@ -100,27 +77,31 @@ public class DataStoreTest {
assertSearch("dog=*lab*dor*", labradorJenny, labradorTim);
// 'in' queries
assertSearch("bird in (eagle, pigeon, flamingo)", eagleTim, pigeonJennifer, flamingoJennifer);
assertSearch("dog in (labrador) and name in (Tim, Jennifer)", labradorTim);
assertSearch("name in (Jenn*)", pigeonJennifer, flamingoJennifer, labradorJenny);
assertSearch("name in (*) and dog=labrador", labradorJenny, labradorTim);
assertSearch("name in (XYZ, *) and dog=labrador", labradorJenny, labradorTim);
// TODO fix in queries
/*
* assertSearch("bird in (eagle, pigeon, flamingo)", eagleTim, pigeonJennifer,
* flamingoJennifer);
* assertSearch("dog in (labrador) and name in (Tim, Jennifer)", labradorTim);
* assertSearch("name in (Jenn*)", pigeonJennifer, flamingoJennifer,
* labradorJenny); assertSearch("name in (*) and dog=labrador", labradorJenny,
* labradorTim); assertSearch("name in (XYZ, *) and dog=labrador",
* labradorJenny, labradorTim);
*/
}
public void testGetByTags() throws IOException {
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags eagleTim1 = Tags.create("bird", "eagle", "name", "Tim");
final Tags eagleTim2 = Tags.create("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer");
dataStore.createNewFile(eagleTim1);
dataStore.createNewFile(eagleTim2);
dataStore.createNewFile(pigeonJennifer);
dataStore.createNewFile(flamingoJennifer);
tagsToBlockStorageRootBlockNumber.put(eagleTim1, dataStore.createNewFile(eagleTim1));
tagsToBlockStorageRootBlockNumber.put(eagleTim2, dataStore.createNewFile(eagleTim2));
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(flamingoJennifer));
// eagleTim1 and eagleTim2 have the same tags, so we find both docs
final List<Doc> docsEagleTim = dataStore.getByTags(eagleTim1);
@@ -132,40 +113,36 @@ public class DataStoreTest {
private void assertSearch(final String query, final Tags... tags) {
final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
final Path storageBasePath = dataStore.getStorageBasePath();
final List<Path> expectedPaths = CollectionUtils.map(CollectionUtils.map(tags, tagsToPath::get),
storageBasePath::resolve);
final List<Long> expectedPaths = CollectionUtils.map(tags, tagsToBlockStorageRootBlockNumber::get);
Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + getTagsForPaths(actual));
Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + actual);
}
private List<Tags> getTagsForPaths(final List<Path> paths) {
final List<Tags> result = new ArrayList<>();
for (final Path path : paths) {
result.add(getTagForPath(path));
}
return result;
}
private Tags getTagForPath(final Path path) {
for (final Entry<Tags, Path> e : tagsToPath.entrySet()) {
if (e.getValue().equals(path)) {
return e.getKey();
}
}
return null;
}
// private List<Tags> getTagsForPaths(final List<Path> paths) {
//
// final List<Tags> result = new ArrayList<>();
//
// for (final Path path : paths) {
// result.add(getTagForPath(path));
// }
// return result;
// }
//
// private Tags getTagForPath(final Path path) {
// for (final Entry<Tags, Long> e : tagsToBlockStorageRootBlockNumber.entrySet()) {
//
// if (e.getValue().equals(path)) {
// return e.getKey();
// }
// }
// return null;
// }
private void assertSearch(final DataStore dataStore, final String query, final Path... paths) {
final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
final List<Long> actual = CollectionUtils.map(actualDocs, Doc::getRootBlockNumber);
Assert.assertEquals(actual, Arrays.asList(paths));
}

137
data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java
View File

@@ -1,137 +0,0 @@
package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import org.lucares.utils.CollectionUtils;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class FolderStorageTest {
private static final String SUFFIX = ".txt";
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@Test
public void testFolderStructureRespectingToMaxFilesPerFolder() throws Exception {
final int maxFilesPerFolder = 2;
storeFiles(maxFilesPerFolder);
storeFiles(maxFilesPerFolder, "a", "b", "c", "d", "e");
storeFiles(maxFilesPerFolder, "f");
storeFiles(maxFilesPerFolder, "g", "h", "i");
final List<Path> actualFiles = getPathsRelativeToDataDirectory();
final List<Path> expectedFiles = Arrays.asList(//
Paths.get("0", "0", "a$" + SUFFIX), //
Paths.get("0", "0", "b$" + SUFFIX), //
Paths.get("0", "1", "c$" + SUFFIX), //
Paths.get("0", "1", "d$" + SUFFIX), //
Paths.get("1", "0", "e$" + SUFFIX), //
Paths.get("1", "0", "f$" + SUFFIX), //
Paths.get("1", "1", "g$" + SUFFIX), //
Paths.get("1", "1", "h$" + SUFFIX), //
Paths.get("2", "0", "i$" + SUFFIX)// The first level might
// overflow
);
Assert.assertEquals(actualFiles, expectedFiles);
}
@Test
public void testDuplicateNames() throws Exception {
final int maxFilesPerFolder = 3;
storeFiles(maxFilesPerFolder, "a", "a", "a", "a");
final List<Path> actualFiles = getPathsRelativeToDataDirectory();
final List<Path> expectedFiles = Arrays.asList(//
Paths.get("0", "0", "a$" + SUFFIX), //
Paths.get("0", "0", "a$1" + SUFFIX), //
Paths.get("0", "0", "a$2" + SUFFIX), //
Paths.get("0", "1", "a$" + SUFFIX)//
);
Assert.assertEquals(actualFiles, expectedFiles);
}
@Test
public void testCreateAndUpdateFileListing() throws Exception {
final int maxFilesPerFolder = 10;
// initial creation
{
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
storage.insert("abc", ".txt");
storage.insert("def", ".txt");
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null)));
}
}
// load existing storage
{
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
// files inserted previously are still there
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null)));
// add new file
storage.insert("ghi", ".txt");
// listing is updated
final List<ListingFileEntry> updatedListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(updatedListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null), //
new ListingFileEntry("ghi$.txt", 26, null)));
}
}
}
private List<Path> getPathsRelativeToDataDirectory() throws IOException {
List<Path> actualFiles = FileUtils.listRecursively(dataDirectory);
actualFiles = CollectionUtils.filter(actualFiles,
p -> !p.getFileName().toString().equals(FolderStorage.LISTING_FILE_NAME));
CollectionUtils.mapInPlace(actualFiles, p -> dataDirectory.relativize(p));
Collections.sort(actualFiles);
return actualFiles;
}
private void storeFiles(final int maxFilesPerFolder, final String... filenames) throws IOException {
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder)) {
for (final String filename : filenames) {
storage.insert(filename, SUFFIX);
}
}
}
}

24
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -5,9 +5,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbDB;
@@ -24,7 +22,6 @@ public class ProposerTest {
private Path dataDirectory;
private PdbDB db;
private Map<Tags, Path> tagsToPath;
@BeforeClass
public void beforeClass() throws Exception {
@@ -37,14 +34,12 @@ public class ProposerTest {
FileUtils.delete(dataDirectory);
db.close();
db = null;
tagsToPath = null;
Tags.STRING_COMPRESSOR = null;
}
private void initDatabase() throws Exception {
db = new PdbDB(dataDirectory);
tagsToPath = new LinkedHashMap<>();
final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
@@ -52,17 +47,12 @@ public class ProposerTest {
final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim");
tagsToPath.put(eagleTim, null);
tagsToPath.put(eagleTimothy, null);
tagsToPath.put(pigeonJennifer, null);
tagsToPath.put(flamingoJennifer, null);
tagsToPath.put(labradorJenny, null);
tagsToPath.put(labradorTim, null);
for (final Tags tags : tagsToPath.keySet()) {
final Path newFile = db.createNewFile(tags);
tagsToPath.put(tags, newFile);
}
db.createNewFile(eagleTim);
db.createNewFile(eagleTimothy);
db.createNewFile(pigeonJennifer);
db.createNewFile(flamingoJennifer);
db.createNewFile(labradorJenny);
db.createNewFile(labradorTim);
}
public void testEmptyQuery() throws Exception {
@@ -105,6 +95,8 @@ public class ProposerTest {
*/
}
// TODO fix the in expression
@Test(enabled = false)
public void testInExpressions() throws Exception {
assertProposals("name in (Timothy,)", 17, //
new Proposal("Jennifer", "name in (Timothy,Jennifer)", true, "name in (Timothy,Jennifer)", 25), //