andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use RandomAccessFile in FolderStorage.getPathByOffset()

Browse Source 
The old implementation opened a new buffered reader everytime
getPathByOffset was called. This took 1/20th of a second or
longer. For queries that visited thousands of files this could
take a long time.
We are now using a RandomAccessFile, that is opened once. The
average time spend in getPathByOffset is now down to 0.11ms.
This commit is contained in:
Andreas Huber
2018-05-10 10:22:25 +02:00
parent 82b8a8a932
commit 911062e26b
18 changed files with 215 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -10,6 +10,7 @@ import org.lucares.pdb.datastore.internal.DataStore;
public class Doc { public class Doc {
private final Tags tags; private final Tags tags;
private final long offsetInListingFile; private final long offsetInListingFile;
private final Path storageBasePath;
private byte[] path; private byte[] path;
/** /**
@@ -24,21 +25,25 @@ public class Doc {
* @param tags * @param tags
* @param offsetInListingFile * @param offsetInListingFile
* must be set if {@code path} is {@code null} * must be set if {@code path} is {@code null}
* @param path * @param storageBasePath
* optional, can be {@code null} * the storage base path.
* @param relativePath
* optional, can be {@code null}. This path is relative to
* {@code storageBasePath}
*/ */
public Doc(final Tags tags, final long offsetInListingFile, final Path path) { public Doc(final Tags tags, final long offsetInListingFile, final Path storageBasePath, final Path relativePath) {
super(); super();
this.tags = tags; this.tags = tags;
this.offsetInListingFile = offsetInListingFile; this.offsetInListingFile = offsetInListingFile;
setPath(path); this.storageBasePath = storageBasePath;
setRelativePath(relativePath);
} }
public Tags getTags() { public Tags getTags() {
return tags; return tags;
} }
public void setPath(final Path path) { public void setRelativePath(final Path path) {
if (path != null) { if (path != null) {
this.path = path.toString().getBytes(StandardCharsets.UTF_8); this.path = path.toString().getBytes(StandardCharsets.UTF_8);
} else { } else {
@@ -54,18 +59,18 @@ public class Doc {
* *
* @return the path * @return the path
*/ */
public Path getPath(final FolderStoragePathResolver resolver) { public Path getAbsolutePath(final FolderStoragePathResolver resolver) {
if (path == null) { if (path == null) {
final Path resolvedPath = resolver.getPath(offsetInListingFile); final Path resolvedPath = resolver.getPath(offsetInListingFile);
setPath(resolvedPath); setRelativePath(resolvedPath);
} }
final Path result = Paths.get(new String(path, StandardCharsets.UTF_8)); final Path relativePath = Paths.get(new String(path, StandardCharsets.UTF_8));
return result; return storageBasePath.resolve(relativePath);
} }
private Path getPathNullable() { private Path getAbsolutePathNullable() {
return getPath(FolderStoragePathResolver.NULL); return getAbsolutePath(FolderStoragePathResolver.NULL);
} }
public long getOffsetInListingFile() { public long getOffsetInListingFile() {
@@ -74,7 +79,7 @@ public class Doc {
@Override @Override
public String toString() { public String toString() {
return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getPathNullable() return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getAbsolutePathNullable()
+ "]"; + "]";
} }

11
data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java
View File

@@ -9,7 +9,7 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.internal.DataStore; import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.datastore.internal.Proposer; import org.lucares.pdb.datastore.internal.Proposer;
public class PdbDB { public class PdbDB implements AutoCloseable {
private final DataStore dataStore; private final DataStore dataStore;
private final Proposer proposer; private final Proposer proposer;
@@ -47,4 +47,13 @@ public class PdbDB {
public FolderStoragePathResolver getFolderStoragePathResolver() { public FolderStoragePathResolver getFolderStoragePathResolver() {
return dataStore.getFolderStoragePathResolver(); return dataStore.getFolderStoragePathResolver();
} }
public Path getStorageBasePath() {
return dataStore.getStorageBasePath();
}
@Override
public void close() throws IOException {
dataStore.close();
}
} }

20
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -25,7 +25,7 @@ import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class DataStore { public class DataStore implements AutoCloseable {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery"); .getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger INITIALIZE = LoggerFactory.getLogger("org.lucares.metrics.dataStore.init"); private static final Logger INITIALIZE = LoggerFactory.getLogger("org.lucares.metrics.dataStore.init");
@@ -43,11 +43,13 @@ public class DataStore {
private final FolderStorage folderStorage; private final FolderStorage folderStorage;
private final FolderStoragePathResolver folderStoragePathResolver; private final FolderStoragePathResolver folderStoragePathResolver;
private final Path storageBasePath;
public DataStore(final Path dataDirectory) throws IOException { public DataStore(final Path dataDirectory) throws IOException {
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(dataDirectory)); Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(dataDirectory));
folderStorage = new FolderStorage(storageDirectory(dataDirectory), 1000); storageBasePath = storageDirectory(dataDirectory);
folderStorage = new FolderStorage(storageBasePath, 1000);
init(folderStorage); init(folderStorage);
folderStoragePathResolver = folderStorage::getPathByOffset; folderStoragePathResolver = folderStorage::getPathByOffset;
@@ -60,7 +62,7 @@ public class DataStore {
files// .parallel() files// .parallel()
.forEach(listingFileEntry -> { .forEach(listingFileEntry -> {
listingFileEntry.unsetPath(); // unset the path, so that we don't store it for every document (will listingFileEntry.unsetRelativePath(); // unset the path, so that we don't store it for every document (will
// be // be
// initialized lazily if needed) // initialized lazily if needed)
@@ -80,7 +82,8 @@ public class DataStore {
private void cacheTagToFileMapping(final Tags tags, final ListingFileEntry listingFileEntry) { private void cacheTagToFileMapping(final Tags tags, final ListingFileEntry listingFileEntry) {
final int docId; final int docId;
final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), listingFileEntry.getPath()); final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), storageBasePath,
listingFileEntry.getPath());
synchronized (docIdToDoc) { synchronized (docIdToDoc) {
docId = docIdToDoc.size(); docId = docIdToDoc.size();
docIdToDoc.add(newDoc); docIdToDoc.add(newDoc);
@@ -245,4 +248,13 @@ public class DataStore {
public FolderStoragePathResolver getFolderStoragePathResolver() { public FolderStoragePathResolver getFolderStoragePathResolver() {
return folderStoragePathResolver; return folderStoragePathResolver;
} }
public Path getStorageBasePath() {
return storageBasePath;
}
@Override
public void close() throws IOException {
folderStorage.close();
}
} }

87
data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java
View File

@@ -1,7 +1,5 @@
package org.lucares.pdb.datastore.internal; package org.lucares.pdb.datastore.internal;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.io.Writer; import java.io.Writer;
@@ -22,12 +20,15 @@ import org.lucares.pdb.api.RuntimeIOException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
public class FolderStorage { public class FolderStorage implements AutoCloseable {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.US_ASCII);
static final String LISTING_FILE_NAME = "listing.csv"; static final String LISTING_FILE_NAME = "listing.csv";
private final static Logger LOGGER = LoggerFactory.getLogger(FolderStorage.class); private final static Logger LOGGER = LoggerFactory.getLogger(FolderStorage.class);
private final static Logger METRICS_CREATE_LISTING_FILE = LoggerFactory private final static Logger METRICS_CREATE_LISTING_FILE = LoggerFactory
.getLogger("org.lucares.metrics.fodlerStorage.createListingFile"); .getLogger("org.lucares.metrics.folderStorage.createListingFile");
private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.getPathByOffset");
private final Path storageBaseDirectory; private final Path storageBaseDirectory;
@@ -39,14 +40,21 @@ public class FolderStorage {
private final int maxFilesPerFolder; private final int maxFilesPerFolder;
private final Path listingFile; private final Path listingFilePath;
private final RandomAccessFile listingFile;
public FolderStorage(final Path storageBaseDirectory, final int maxFilesPerFolder) throws IOException { public FolderStorage(final Path storageBaseDirectory, final int maxFilesPerFolder) throws IOException {
this.storageBaseDirectory = storageBaseDirectory; this.storageBaseDirectory = storageBaseDirectory;
this.listingFile = storageBaseDirectory.resolve(LISTING_FILE_NAME); this.listingFilePath = storageBaseDirectory.resolve(LISTING_FILE_NAME);
this.maxFilesPerFolder = maxFilesPerFolder; this.maxFilesPerFolder = maxFilesPerFolder;
init(); init();
initListingFileIfNotExists(); initListingFileIfNotExists();
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws");
}
@Override
public void close() throws IOException {
listingFile.close();
} }
private void init() throws IOException { private void init() throws IOException {
@@ -85,23 +93,19 @@ public class FolderStorage {
} }
private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException { private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException {
final long offsetInListingFile = getFilePointer(); final long offsetInListingFile = Files.size(listingFilePath);
try (Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE, // remember: all paths within storageBaseDirectory use only ascii characters
StandardOpenOption.APPEND)) { try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE,
out.write(newFile.toString()); StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
out.write("\n");
final Path relativePath = storageBaseDirectory.relativize(newFile);
listingFile.seek(offsetInListingFile);
listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE);
} }
final String filename = newFile.getFileName().toString(); final String filename = newFile.getFileName().toString();
return new ListingFileEntry(filename, offsetInListingFile, newFile); final Path relativePath = storageBaseDirectory.relativize(newFile);
} return new ListingFileEntry(filename, offsetInListingFile, relativePath);
private long getFilePointer() throws FileNotFoundException, IOException {
final RandomAccessFile randomAccessFile = new RandomAccessFile(listingFile.toFile(), "r");
try {
return randomAccessFile.getFilePointer();
} finally {
randomAccessFile.close();
}
} }
private void ensureCapacity() throws IOException { private void ensureCapacity() throws IOException {
@@ -125,21 +129,15 @@ public class FolderStorage {
public Stream<ListingFileEntry> list() throws IOException { public Stream<ListingFileEntry> list() throws IOException {
return readListingFile(); final ListingFileIterator iterator = new ListingFileIterator(listingFilePath);
} final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
private Stream<ListingFileEntry> readListingFile() throws IOException { final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
try (final ListingFileIterator iterator = new ListingFileIterator(listingFile)) {
final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
}
} }
private void initListingFileIfNotExists() throws IOException { private void initListingFileIfNotExists() throws IOException {
if (!Files.exists(listingFile)) { if (!Files.exists(listingFilePath)) {
final long start = System.nanoTime(); final long start = System.nanoTime();
LOGGER.info("listing file not found -> creating a new one"); LOGGER.info("listing file not found -> creating a new one");
createNewListingFile(); createNewListingFile();
@@ -151,29 +149,38 @@ public class FolderStorage {
final int maxDepth = Integer.MAX_VALUE; final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matchRegularFiles = (path, attr) -> Files.isRegularFile(path); final BiPredicate<Path, BasicFileAttributes> matchRegularFiles = (path, attr) -> Files.isRegularFile(path);
try (final Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE, // remember: all paths within storageBaseDirectory use only ascii characters
StandardOpenOption.APPEND); try (final Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII,
StandardOpenOption.CREATE, StandardOpenOption.APPEND);
final Stream<Path> stream = Files.find(storageBaseDirectory, maxDepth, matchRegularFiles)) { final Stream<Path> stream = Files.find(storageBaseDirectory, maxDepth, matchRegularFiles)) {
final Iterator<Path> iterator = stream.iterator(); final Iterator<Path> iterator = stream.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
final Path path = iterator.next(); final Path path = iterator.next();
if (!path.getFileName().toString().equals(LISTING_FILE_NAME)) { if (!path.getFileName().toString().equals(LISTING_FILE_NAME)) {
out.write(path.toString());
final Path relativePath = storageBaseDirectory.relativize(path);
out.write(relativePath.toString());
out.write("\n"); out.write("\n");
} }
} }
} }
} }
public Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException { public synchronized Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException {
try (BufferedReader reader = Files.newBufferedReader(listingFile, StandardCharsets.UTF_8)) { final long start = System.nanoTime();
reader.skip(offsetInListingFile); try {
final String line = reader.readLine(); listingFile.seek(offsetInListingFile);
// remember: all paths within storageBaseDirectory use only ascii characters
final String line = listingFile.readLine();
return Paths.get(line); return Paths.get(line);
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} finally {
METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms");
} }
} }

38
data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java
View File

@@ -4,15 +4,29 @@ import java.nio.file.Path;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import org.lucares.pdb.datastore.Doc;
public class ListingFileEntry { public class ListingFileEntry {
private final String filename; private final String filename;
private final long offsetInListingFile; private final long offsetInListingFile;
private Path path; private Path relativePath;
public ListingFileEntry(final String filename, final long offsetInListingFile, final Path path) { /**
* Create a new {@link ListingFileEntry}.
* <p>
* The {@code path} is optional. When the {@link ListingFileEntry} is read from
* the listing file, then the {@code path} is set to {@code null}. This is done
* to save memory. See {@link Doc} for more information on its usage.
*
* @param filename
* @param offsetInListingFile
* @param relativePath
* optional, see {@link Doc}
*/
public ListingFileEntry(final String filename, final long offsetInListingFile, final Path relativePath) {
this.filename = filename; this.filename = filename;
this.offsetInListingFile = offsetInListingFile; this.offsetInListingFile = offsetInListingFile;
this.path = path; this.relativePath = relativePath;
} }
public String getFilename() { public String getFilename() {
@@ -23,19 +37,19 @@ public class ListingFileEntry {
return offsetInListingFile; return offsetInListingFile;
} }
public void unsetPath() { public void unsetRelativePath() {
path = null; relativePath = null;
} }
@Nullable @Nullable
public Path getPath() { public Path getPath() {
return path; return relativePath;
} }
@Override @Override
public String toString() { public String toString() {
return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile + ", path=" return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile
+ path + "]"; + ", relativePath=" + relativePath + "]";
} }
@Override @Override
@@ -44,7 +58,7 @@ public class ListingFileEntry {
int result = 1; int result = 1;
result = prime * result + ((filename == null) ? 0 : filename.hashCode()); result = prime * result + ((filename == null) ? 0 : filename.hashCode());
result = prime * result + (int) (offsetInListingFile ^ (offsetInListingFile >>> 32)); result = prime * result + (int) (offsetInListingFile ^ (offsetInListingFile >>> 32));
result = prime * result + ((path == null) ? 0 : path.hashCode()); result = prime * result + ((relativePath == null) ? 0 : relativePath.hashCode());
return result; return result;
} }
@@ -64,10 +78,10 @@ public class ListingFileEntry {
return false; return false;
if (offsetInListingFile != other.offsetInListingFile) if (offsetInListingFile != other.offsetInListingFile)
return false; return false;
if (path == null) { if (relativePath == null) {
if (other.path != null) if (other.relativePath != null)
return false; return false;
} else if (!path.equals(other.path)) } else if (!relativePath.equals(other.relativePath))
return false; return false;
return true; return true;
} }

14
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -42,14 +42,16 @@ public class DataStoreTest {
final Path path; final Path path;
{ {
final DataStore dataStore = new DataStore(dataDirectory); final DataStore dataStore = new DataStore(dataDirectory);
final Path storageBasePath = dataStore.getStorageBasePath();
final Tags tags = Tags.create("key1", "value1", "key2", "value2"); final Tags tags = Tags.create("key1", "value1", "key2", "value2");
path = dataStore.createNewFile(tags); path = dataStore.createNewFile(tags);
assertSearch(dataStore, "key1=value1", path); assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
} }
{ {
final DataStore dataStore = new DataStore(dataDirectory); final DataStore dataStore = new DataStore(dataDirectory);
assertSearch(dataStore, "key1=value1", path); final Path storageBasePath = dataStore.getStorageBasePath();
assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
} }
} }
@@ -124,9 +126,11 @@ public class DataStoreTest {
private void assertSearch(final String query, final Tags... tags) { private void assertSearch(final String query, final Tags... tags) {
final List<Doc> actualDocs = dataStore.search(query); final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs, final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getPath(dataStore.getFolderStoragePathResolver())); doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
final List<Path> expectedPaths = CollectionUtils.map(tags, tagsToPath::get); final Path storageBasePath = dataStore.getStorageBasePath();
final List<Path> expectedPaths = CollectionUtils.map(CollectionUtils.map(tags, tagsToPath::get),
storageBasePath::resolve);
Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + getTagsForPaths(actual)); Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + getTagsForPaths(actual));
} }
@@ -154,7 +158,7 @@ public class DataStoreTest {
private void assertSearch(final DataStore dataStore, final String query, final Path... paths) { private void assertSearch(final DataStore dataStore, final String query, final Path... paths) {
final List<Doc> actualDocs = dataStore.search(query); final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs, final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getPath(dataStore.getFolderStoragePathResolver())); doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
Assert.assertEquals(actual, Arrays.asList(paths)); Assert.assertEquals(actual, Arrays.asList(paths));
} }

53
data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java
View File

@@ -79,40 +79,40 @@ public class FolderStorageTest {
@Test @Test
public void testCreateAndUpdateFileListing() throws Exception { public void testCreateAndUpdateFileListing() throws Exception {
final int maxFilesPerFolder = 10; final int maxFilesPerFolder = 10;
final Path storageLeafFolder = dataDirectory.resolve("0").resolve("0");
final int storageLeafFolderLength = storageLeafFolder.toString().length();
// initial creation // initial creation
{ {
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
storage.insert("abc", ".txt"); storage.insert("abc", ".txt");
storage.insert("def", ".txt"); storage.insert("def", ".txt");
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList()); final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(// Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), // new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null))); new ListingFileEntry("def$.txt", 13, null)));
}
} }
// load existing storage // load existing storage
{ {
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
// files inserted previously are still there // files inserted previously are still there
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList()); final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(// Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), // new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null))); new ListingFileEntry("def$.txt", 13, null)));
// add new file // add new file
storage.insert("ghi", ".txt"); storage.insert("ghi", ".txt");
// listing is updated // listing is updated
final List<ListingFileEntry> updatedListing = storage.list().collect(Collectors.toList()); final List<ListingFileEntry> updatedListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(updatedListing, Arrays.asList(// Assert.assertEquals(updatedListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), // new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null), // new ListingFileEntry("def$.txt", 13, null), //
new ListingFileEntry("ghi$.txt", 2 * storageLeafFolderLength + 20, null))); new ListingFileEntry("ghi$.txt", 26, null)));
}
} }
} }
@@ -127,10 +127,11 @@ public class FolderStorageTest {
} }
private void storeFiles(final int maxFilesPerFolder, final String... filenames) throws IOException { private void storeFiles(final int maxFilesPerFolder, final String... filenames) throws IOException {
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder)) {
for (final String filename : filenames) { for (final String filename : filenames) {
storage.insert(filename, SUFFIX); storage.insert(filename, SUFFIX);
}
} }
} }
} }

1
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -34,6 +34,7 @@ public class ProposerTest {
@AfterClass @AfterClass
public void afterClass() throws IOException { public void afterClass() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
db.close();
db = null; db = null;
tagsToPath = null; tagsToPath = null;
Tags.STRING_COMPRESSOR = null; Tags.STRING_COMPRESSOR = null;

20
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
@@ -16,18 +17,18 @@ import org.slf4j.LoggerFactory;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable { public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = LoggerFactory private final static Logger LOGGER = LoggerFactory.getLogger(PdbFileIterator.class);
.getLogger(PdbFileIterator.class);
private static final class EntrySupplier implements Supplier<Entry>, private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
AutoCloseable {
private final Queue<PdbFile> pdbFiles; private final Queue<PdbFile> pdbFiles;
private PdbReader reader; private PdbReader reader;
private PdbFile currentPdbFile; private PdbFile currentPdbFile;
private final Path storageBasePath;
public EntrySupplier(final Collection<PdbFile> pdbFiles) { public EntrySupplier(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
super(); super();
this.storageBasePath = storageBasePath;
this.pdbFiles = new ArrayDeque<>(pdbFiles); this.pdbFiles = new ArrayDeque<>(pdbFiles);
} }
@@ -73,14 +74,13 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
try { try {
if (Files.size(currentPdbFile.getPath()) > 0) { if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile); reader = new PdbReader(storageBasePath, currentPdbFile);
break; break;
} else { } else {
LOGGER.info("ignoring empty file " + currentPdbFile); LOGGER.info("ignoring empty file " + currentPdbFile);
} }
} catch (final FileNotFoundException e) { } catch (final FileNotFoundException e) {
LOGGER.warn("the pdbFile " + currentPdbFile.getPath() LOGGER.warn("the pdbFile " + currentPdbFile.getPath() + " is missing", e);
+ " is missing", e);
} catch (final IOException e) { } catch (final IOException e) {
throw new ReadException(e); throw new ReadException(e);
} }
@@ -100,8 +100,8 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private Optional<Entry> next = Optional.empty(); private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) { public PdbFileIterator(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(pdbFiles); supplier = new EntrySupplier(storageBasePath, pdbFiles);
} }
@Override @Override

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java
View File

@@ -2,12 +2,14 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
class PdbFileUtils { class PdbFileUtils {
static OffsetDateTime dateOffset(final PdbFile pdbFile) throws FileNotFoundException, IOException { static OffsetDateTime dateOffset(final Path storageBasePath, final PdbFile pdbFile)
throws FileNotFoundException, IOException {
try (PdbReader reader = new PdbReader(pdbFile)) { try (PdbReader reader = new PdbReader(storageBasePath, pdbFile)) {
reader.seekToLastValue(); reader.seekToLastValue();
return reader.getDateOffsetAtCurrentPosition(); return reader.getDateOffsetAtCurrentPosition();
} }

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
@@ -11,10 +12,11 @@ public class PdbFileViewer {
public static void main(final String[] args) throws FileNotFoundException, IOException { public static void main(final String[] args) throws FileNotFoundException, IOException {
final File file = new File(args[0]); final File file = new File(args[0]);
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); final Path baseDirectory = file.toPath().getParent();
final PdbFile pdbFile = new PdbFile(file.toPath().getFileName(), TAGS);
long countMeasurements = 0; long countMeasurements = 0;
try (final PdbReader reader = new PdbReader(pdbFile, false)) { try (final PdbReader reader = new PdbReader(baseDirectory, pdbFile, false)) {
long value = 0; long value = 0;
int nextByte; int nextByte;

9
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.file.Path;
import java.time.Instant; import java.time.Instant;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneId; import java.time.ZoneId;
@@ -26,15 +27,15 @@ class PdbReader implements AutoCloseable {
private final PdbFile pdbFile; private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws ReadException { public PdbReader(final Path storageBasePath, final PdbFile pdbFile) throws ReadException {
this(pdbFile, true); this(storageBasePath, pdbFile, true);
} }
PdbReader(final PdbFile pdbFile, final boolean initialize) throws ReadException { PdbReader(final Path storageBasePath, final PdbFile pdbFile, final boolean initialize) throws ReadException {
super(); super();
try { try {
this.pdbFile = pdbFile; this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile(); final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile)); this.data = new BufferedInputStream(new FileInputStream(storageFile));

16
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -6,6 +6,7 @@ import java.io.FileOutputStream;
import java.io.Flushable; import java.io.Flushable;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.nio.file.Path;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Entry;
@@ -80,15 +81,15 @@ class PdbWriter implements AutoCloseable, Flushable {
private final PdbFile pdbFile; private final PdbFile pdbFile;
private long lastEpochMilli; private long lastEpochMilli;
PdbWriter(final PdbFile pdbFile) throws IOException { PdbWriter(final Path storageBasePath, final PdbFile pdbFile) throws IOException {
this.pdbFile = pdbFile; this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile(); final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND)); this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
if (storageFile.exists() && storageFile.length() > 0) { if (storageFile.exists() && storageFile.length() > 0) {
// TODO @ahr check version // TODO @ahr check version
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(pdbFile); final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(storageBasePath, pdbFile);
lastEpochMilli = dateOffset.toInstant().toEpochMilli(); lastEpochMilli = dateOffset.toInstant().toEpochMilli();
} else { } else {
writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream); writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream);
@@ -167,16 +168,17 @@ class PdbWriter implements AutoCloseable, Flushable {
output.write(buffer, index, buffer.length - index); output.write(buffer, index, buffer.length - index);
} }
public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException { public static void writeEntry(final Path storageBasePath, final PdbFile pdbFile, final Entry... entries)
try (PdbWriter writer = new PdbWriter(pdbFile)) { throws IOException {
try (PdbWriter writer = new PdbWriter(storageBasePath, pdbFile)) {
for (final Entry entry : entries) { for (final Entry entry : entries) {
writer.write(entry); writer.write(entry);
} }
} }
} }
public static void init(final PdbFile result) throws IOException { public static void init(final Path storageBasePath, final PdbFile result) throws IOException {
writeEntry(result); writeEntry(storageBasePath, result);
} }
@Override @Override

7
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -171,7 +171,7 @@ public class PerformanceDb implements AutoCloseable {
} }
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) { private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final PdbFileIterator iterator = new PdbFileIterator(pdbFiles); final PdbFileIterator iterator = new PdbFileIterator(db.getStorageBasePath(), pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
final Stream<Entry> stream = StreamSupport.stream(spliterator, false); final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
@@ -188,6 +188,11 @@ public class PerformanceDb implements AutoCloseable {
@Override @Override
public void close() { public void close() {
tagsToFile.close(); tagsToFile.close();
try {
db.close();
} catch (final IOException e) {
LOGGER.error("failed to close PdbDB", e);
}
} }
public List<Proposal> autocomplete(final String query, final int caretIndex) { public List<Proposal> autocomplete(final String query, final int caretIndex) {

6
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -72,7 +72,7 @@ public class TagsToFile implements AutoCloseable {
for (final Doc document : searchResult) { for (final Doc document : searchResult) {
final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver(); final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver();
final Path path = document.getPath(resolver); final Path path = document.getAbsolutePath(resolver);
final Tags tags = document.getTags(); final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(path, tags); final PdbFile pdbFile = new PdbFile(path, tags);
@@ -160,7 +160,7 @@ public class TagsToFile implements AutoCloseable {
final long start = System.nanoTime(); final long start = System.nanoTime();
try { try {
final PdbFile pdbFile = createNewPdbFile(tags); final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile); final PdbWriter result = new PdbWriter(db.getStorageBasePath(), pdbFile);
getOrInit(tags).addWriter(result); getOrInit(tags).addWriter(result);
@@ -179,7 +179,7 @@ public class TagsToFile implements AutoCloseable {
final Path storageFile = db.createNewFile(tags); final Path storageFile = db.createNewFile(tags);
final PdbFile result = new PdbFile(storageFile, tags); final PdbFile result = new PdbFile(storageFile, tags);
PdbWriter.init(result); PdbWriter.init(db.getStorageBasePath(), result);
return result; return result;
} }

14
performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java
View File

@@ -73,15 +73,16 @@ public class PdbReaderWriterTest {
public void testWriteRead(final List<Entry> entries) throws Exception { public void testWriteRead(final List<Entry> entries) throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) { try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
for (final Entry entry : entries) { for (final Entry entry : entries) {
writer.write(entry); writer.write(entry);
} }
} }
try (final PdbReader reader = new PdbReader(pdbFile)) { try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
for (final Entry entry : entries) { for (final Entry entry : entries) {
@@ -101,9 +102,10 @@ public class PdbReaderWriterTest {
final Entry entryA = new Entry(1, 1, TAGS); final Entry entryA = new Entry(1, 1, TAGS);
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) { try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
writer.write(entryA); writer.write(entryA);
} }
@@ -116,7 +118,7 @@ public class PdbReaderWriterTest {
Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND); Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND);
try (final PdbReader reader = new PdbReader(pdbFile)) { try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError()); final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actualA, entryA); Assert.assertEquals(actualA, entryA);

5
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -111,7 +111,8 @@ public class PerformanceDbTest {
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList(); final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries); Assert.assertEquals(actualEntries, entries);
final List<Path> filesInStorage = FileUtils.listRecursively(DataStore.storageDirectory(dataDirectory)); final Path storageBasePath = DataStore.storageDirectory(dataDirectory);
final List<Path> filesInStorage = FileUtils.listRecursively(storageBasePath);
Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv"); Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv");
@@ -119,7 +120,7 @@ public class PerformanceDbTest {
final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags); final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags);
try (PdbReader pdbReader = new PdbReader(pdbFile)) { try (PdbReader pdbReader = new PdbReader(storageBasePath, pdbFile)) {
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0)); Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0));
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1)); Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1));
Assert.assertEquals(pdbReader.readEntry().isPresent(), false); Assert.assertEquals(pdbReader.readEntry().isPresent(), false);

17
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -31,8 +31,8 @@ public class TagsToFilesTest {
public void test() throws Exception { public void test() throws Exception {
final PdbDB db = new PdbDB(dataDirectory); try (final PdbDB db = new PdbDB(dataDirectory); //
try (final TagsToFile tagsToFile = new TagsToFile(db)) { final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue"); final Tags tags = Tags.create("myKey", "myValue");
@@ -46,8 +46,9 @@ public class TagsToFilesTest {
} }
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception { public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) { try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1); final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1); final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
@@ -66,8 +67,8 @@ public class TagsToFilesTest {
@Test(invocationCount = 1) @Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception { public void testNewFileIfDateIsTooOld() throws Exception {
final PdbDB db = new PdbDB(dataDirectory); try (final PdbDB db = new PdbDB(dataDirectory); //
try (final TagsToFile tagsToFile = new TagsToFile(db);) { final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1); final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -104,8 +105,8 @@ public class TagsToFilesTest {
public void testIdenticalDatesGoIntoSameFile() throws Exception { public void testIdenticalDatesGoIntoSameFile() throws Exception {
final PdbDB db = new PdbDB(dataDirectory); try (final PdbDB db = new PdbDB(dataDirectory); //
try (final TagsToFile tagsToFile = new TagsToFile(db)) { final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);