andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use RandomAccessFile in FolderStorage.getPathByOffset()

Browse Source 
The old implementation opened a new buffered reader everytime
getPathByOffset was called. This took 1/20th of a second or
longer. For queries that visited thousands of files this could
take a long time.
We are now using a RandomAccessFile, that is opened once. The
average time spend in getPathByOffset is now down to 0.11ms.
This commit is contained in:
Andreas Huber
2018-05-10 10:22:25 +02:00
parent 82b8a8a932
commit 911062e26b
18 changed files with 215 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
data-store/src/main/java/org/lucares/pdb/datastore/Doc.java
View File

@@ -10,6 +10,7 @@ import org.lucares.pdb.datastore.internal.DataStore;
public class Doc {
private final Tags tags;
private final long offsetInListingFile;
private final Path storageBasePath;
private byte[] path;
/**
@@ -24,21 +25,25 @@ public class Doc {
* @param tags
* @param offsetInListingFile
* must be set if {@code path} is {@code null}
* @param path
* optional, can be {@code null}
* @param storageBasePath
* the storage base path.
* @param relativePath
* optional, can be {@code null}. This path is relative to
* {@code storageBasePath}
*/
public Doc(final Tags tags, final long offsetInListingFile, final Path path) {
public Doc(final Tags tags, final long offsetInListingFile, final Path storageBasePath, final Path relativePath) {
super();
this.tags = tags;
this.offsetInListingFile = offsetInListingFile;
setPath(path);
this.storageBasePath = storageBasePath;
setRelativePath(relativePath);
}
public Tags getTags() {
return tags;
}
public void setPath(final Path path) {
public void setRelativePath(final Path path) {
if (path != null) {
this.path = path.toString().getBytes(StandardCharsets.UTF_8);
} else {
@@ -54,18 +59,18 @@ public class Doc {
*
* @return the path
*/
public Path getPath(final FolderStoragePathResolver resolver) {
public Path getAbsolutePath(final FolderStoragePathResolver resolver) {
if (path == null) {
final Path resolvedPath = resolver.getPath(offsetInListingFile);
setPath(resolvedPath);
setRelativePath(resolvedPath);
}
final Path result = Paths.get(new String(path, StandardCharsets.UTF_8));
return result;
final Path relativePath = Paths.get(new String(path, StandardCharsets.UTF_8));
return storageBasePath.resolve(relativePath);
}
private Path getPathNullable() {
return getPath(FolderStoragePathResolver.NULL);
private Path getAbsolutePathNullable() {
return getAbsolutePath(FolderStoragePathResolver.NULL);
}
public long getOffsetInListingFile() {
@@ -74,7 +79,7 @@ public class Doc {
@Override
public String toString() {
return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getPathNullable()
return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getAbsolutePathNullable()
+ "]";
}

11
data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java
View File

@@ -9,7 +9,7 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.datastore.internal.Proposer;
public class PdbDB {
public class PdbDB implements AutoCloseable {
private final DataStore dataStore;
private final Proposer proposer;
@@ -47,4 +47,13 @@ public class PdbDB {
public FolderStoragePathResolver getFolderStoragePathResolver() {
return dataStore.getFolderStoragePathResolver();
}
public Path getStorageBasePath() {
return dataStore.getStorageBasePath();
}
@Override
public void close() throws IOException {
dataStore.close();
}
}

20
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -25,7 +25,7 @@ import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore {
public class DataStore implements AutoCloseable {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger INITIALIZE = LoggerFactory.getLogger("org.lucares.metrics.dataStore.init");
@@ -43,11 +43,13 @@ public class DataStore {
private final FolderStorage folderStorage;
private final FolderStoragePathResolver folderStoragePathResolver;
private final Path storageBasePath;
public DataStore(final Path dataDirectory) throws IOException {
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(dataDirectory));
folderStorage = new FolderStorage(storageDirectory(dataDirectory), 1000);
storageBasePath = storageDirectory(dataDirectory);
folderStorage = new FolderStorage(storageBasePath, 1000);
init(folderStorage);
folderStoragePathResolver = folderStorage::getPathByOffset;
@@ -60,7 +62,7 @@ public class DataStore {
files// .parallel()
.forEach(listingFileEntry -> {
listingFileEntry.unsetPath(); // unset the path, so that we don't store it for every document (will
listingFileEntry.unsetRelativePath(); // unset the path, so that we don't store it for every document (will
// be
// initialized lazily if needed)
@@ -80,7 +82,8 @@ public class DataStore {
private void cacheTagToFileMapping(final Tags tags, final ListingFileEntry listingFileEntry) {
final int docId;
final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), listingFileEntry.getPath());
final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), storageBasePath,
listingFileEntry.getPath());
synchronized (docIdToDoc) {
docId = docIdToDoc.size();
docIdToDoc.add(newDoc);
@@ -245,4 +248,13 @@ public class DataStore {
public FolderStoragePathResolver getFolderStoragePathResolver() {
return folderStoragePathResolver;
}
public Path getStorageBasePath() {
return storageBasePath;
}
@Override
public void close() throws IOException {
folderStorage.close();
}
}

87
data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java
View File

@@ -1,7 +1,5 @@
package org.lucares.pdb.datastore.internal;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Writer;
@@ -22,12 +20,15 @@ import org.lucares.pdb.api.RuntimeIOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FolderStorage {
public class FolderStorage implements AutoCloseable {
private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.US_ASCII);
static final String LISTING_FILE_NAME = "listing.csv";
private final static Logger LOGGER = LoggerFactory.getLogger(FolderStorage.class);
private final static Logger METRICS_CREATE_LISTING_FILE = LoggerFactory
.getLogger("org.lucares.metrics.fodlerStorage.createListingFile");
.getLogger("org.lucares.metrics.folderStorage.createListingFile");
private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.getPathByOffset");
private final Path storageBaseDirectory;
@@ -39,14 +40,21 @@ public class FolderStorage {
private final int maxFilesPerFolder;
private final Path listingFile;
private final Path listingFilePath;
private final RandomAccessFile listingFile;
public FolderStorage(final Path storageBaseDirectory, final int maxFilesPerFolder) throws IOException {
this.storageBaseDirectory = storageBaseDirectory;
this.listingFile = storageBaseDirectory.resolve(LISTING_FILE_NAME);
this.listingFilePath = storageBaseDirectory.resolve(LISTING_FILE_NAME);
this.maxFilesPerFolder = maxFilesPerFolder;
init();
initListingFileIfNotExists();
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws");
}
@Override
public void close() throws IOException {
listingFile.close();
}
private void init() throws IOException {
@@ -85,23 +93,19 @@ public class FolderStorage {
}
private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException {
final long offsetInListingFile = getFilePointer();
try (Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE,
StandardOpenOption.APPEND)) {
out.write(newFile.toString());
out.write("\n");
final long offsetInListingFile = Files.size(listingFilePath);
// remember: all paths within storageBaseDirectory use only ascii characters
try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE,
StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
final Path relativePath = storageBaseDirectory.relativize(newFile);
listingFile.seek(offsetInListingFile);
listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE);
}
final String filename = newFile.getFileName().toString();
return new ListingFileEntry(filename, offsetInListingFile, newFile);
}
private long getFilePointer() throws FileNotFoundException, IOException {
final RandomAccessFile randomAccessFile = new RandomAccessFile(listingFile.toFile(), "r");
try {
return randomAccessFile.getFilePointer();
} finally {
randomAccessFile.close();
}
final Path relativePath = storageBaseDirectory.relativize(newFile);
return new ListingFileEntry(filename, offsetInListingFile, relativePath);
}
private void ensureCapacity() throws IOException {
@@ -125,21 +129,15 @@ public class FolderStorage {
public Stream<ListingFileEntry> list() throws IOException {
return readListingFile();
}
private Stream<ListingFileEntry> readListingFile() throws IOException {
try (final ListingFileIterator iterator = new ListingFileIterator(listingFile)) {
final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
}
final ListingFileIterator iterator = new ListingFileIterator(listingFilePath);
final Spliterator<ListingFileEntry> spliterator = Spliterators.spliteratorUnknownSize(iterator,
Spliterator.ORDERED);
final Stream<ListingFileEntry> stream = StreamSupport.stream(spliterator, false);
return stream;
}
private void initListingFileIfNotExists() throws IOException {
if (!Files.exists(listingFile)) {
if (!Files.exists(listingFilePath)) {
final long start = System.nanoTime();
LOGGER.info("listing file not found -> creating a new one");
createNewListingFile();
@@ -151,29 +149,38 @@ public class FolderStorage {
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matchRegularFiles = (path, attr) -> Files.isRegularFile(path);
try (final Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE,
StandardOpenOption.APPEND);
// remember: all paths within storageBaseDirectory use only ascii characters
try (final Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII,
StandardOpenOption.CREATE, StandardOpenOption.APPEND);
final Stream<Path> stream = Files.find(storageBaseDirectory, maxDepth, matchRegularFiles)) {
final Iterator<Path> iterator = stream.iterator();
while (iterator.hasNext()) {
final Path path = iterator.next();
if (!path.getFileName().toString().equals(LISTING_FILE_NAME)) {
out.write(path.toString());
final Path relativePath = storageBaseDirectory.relativize(path);
out.write(relativePath.toString());
out.write("\n");
}
}
}
}
public Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException {
public synchronized Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException {
try (BufferedReader reader = Files.newBufferedReader(listingFile, StandardCharsets.UTF_8)) {
reader.skip(offsetInListingFile);
final String line = reader.readLine();
final long start = System.nanoTime();
try {
listingFile.seek(offsetInListingFile);
// remember: all paths within storageBaseDirectory use only ascii characters
final String line = listingFile.readLine();
return Paths.get(line);
} catch (final IOException e) {
throw new RuntimeIOException(e);
} finally {
METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms");
}
}

38
data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java
View File

@@ -4,15 +4,29 @@ import java.nio.file.Path;
import javax.annotation.Nullable;
import org.lucares.pdb.datastore.Doc;
public class ListingFileEntry {
private final String filename;
private final long offsetInListingFile;
private Path path;
private Path relativePath;
public ListingFileEntry(final String filename, final long offsetInListingFile, final Path path) {
/**
* Create a new {@link ListingFileEntry}.
* <p>
* The {@code path} is optional. When the {@link ListingFileEntry} is read from
* the listing file, then the {@code path} is set to {@code null}. This is done
* to save memory. See {@link Doc} for more information on its usage.
*
* @param filename
* @param offsetInListingFile
* @param relativePath
* optional, see {@link Doc}
*/
public ListingFileEntry(final String filename, final long offsetInListingFile, final Path relativePath) {
this.filename = filename;
this.offsetInListingFile = offsetInListingFile;
this.path = path;
this.relativePath = relativePath;
}
public String getFilename() {
@@ -23,19 +37,19 @@ public class ListingFileEntry {
return offsetInListingFile;
}
public void unsetPath() {
path = null;
public void unsetRelativePath() {
relativePath = null;
}
@Nullable
public Path getPath() {
return path;
return relativePath;
}
@Override
public String toString() {
return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile + ", path="
+ path + "]";
return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile
+ ", relativePath=" + relativePath + "]";
}
@Override
@@ -44,7 +58,7 @@ public class ListingFileEntry {
int result = 1;
result = prime * result + ((filename == null) ? 0 : filename.hashCode());
result = prime * result + (int) (offsetInListingFile ^ (offsetInListingFile >>> 32));
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + ((relativePath == null) ? 0 : relativePath.hashCode());
return result;
}
@@ -64,10 +78,10 @@ public class ListingFileEntry {
return false;
if (offsetInListingFile != other.offsetInListingFile)
return false;
if (path == null) {
if (other.path != null)
if (relativePath == null) {
if (other.relativePath != null)
return false;
} else if (!path.equals(other.path))
} else if (!relativePath.equals(other.relativePath))
return false;
return true;
}

14
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -42,14 +42,16 @@ public class DataStoreTest {
final Path path;
{
final DataStore dataStore = new DataStore(dataDirectory);
final Path storageBasePath = dataStore.getStorageBasePath();
final Tags tags = Tags.create("key1", "value1", "key2", "value2");
path = dataStore.createNewFile(tags);
assertSearch(dataStore, "key1=value1", path);
assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
}
{
final DataStore dataStore = new DataStore(dataDirectory);
assertSearch(dataStore, "key1=value1", path);
final Path storageBasePath = dataStore.getStorageBasePath();
assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path));
}
}
@@ -124,9 +126,11 @@ public class DataStoreTest {
private void assertSearch(final String query, final Tags... tags) {
final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getPath(dataStore.getFolderStoragePathResolver()));
doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
final List<Path> expectedPaths = CollectionUtils.map(tags, tagsToPath::get);
final Path storageBasePath = dataStore.getStorageBasePath();
final List<Path> expectedPaths = CollectionUtils.map(CollectionUtils.map(tags, tagsToPath::get),
storageBasePath::resolve);
Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + getTagsForPaths(actual));
}
@@ -154,7 +158,7 @@ public class DataStoreTest {
private void assertSearch(final DataStore dataStore, final String query, final Path... paths) {
final List<Doc> actualDocs = dataStore.search(query);
final List<Path> actual = CollectionUtils.map(actualDocs,
doc -> doc.getPath(dataStore.getFolderStoragePathResolver()));
doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver()));
Assert.assertEquals(actual, Arrays.asList(paths));
}

53
data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java
View File

@@ -79,40 +79,40 @@ public class FolderStorageTest {
@Test
public void testCreateAndUpdateFileListing() throws Exception {
final int maxFilesPerFolder = 10;
final Path storageLeafFolder = dataDirectory.resolve("0").resolve("0");
final int storageLeafFolderLength = storageLeafFolder.toString().length();
// initial creation
{
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);
storage.insert("abc", ".txt");
storage.insert("def", ".txt");
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
storage.insert("abc", ".txt");
storage.insert("def", ".txt");
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null)));
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null)));
}
}
// load existing storage
{
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) {
// files inserted previously are still there
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
// files inserted previously are still there
final List<ListingFileEntry> initialListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null)));
Assert.assertEquals(initialListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null)));
// add new file
storage.insert("ghi", ".txt");
// add new file
storage.insert("ghi", ".txt");
// listing is updated
final List<ListingFileEntry> updatedListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(updatedListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null), //
new ListingFileEntry("ghi$.txt", 2 * storageLeafFolderLength + 20, null)));
// listing is updated
final List<ListingFileEntry> updatedListing = storage.list().collect(Collectors.toList());
Assert.assertEquals(updatedListing, Arrays.asList(//
new ListingFileEntry("abc$.txt", 0, null), //
new ListingFileEntry("def$.txt", 13, null), //
new ListingFileEntry("ghi$.txt", 26, null)));
}
}
}
@@ -127,10 +127,11 @@ public class FolderStorageTest {
}
private void storeFiles(final int maxFilesPerFolder, final String... filenames) throws IOException {
final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);
try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder)) {
for (final String filename : filenames) {
storage.insert(filename, SUFFIX);
for (final String filename : filenames) {
storage.insert(filename, SUFFIX);
}
}
}
}

1
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -34,6 +34,7 @@ public class ProposerTest {
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
db.close();
db = null;
tagsToPath = null;
Tags.STRING_COMPRESSOR = null;

20
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
@@ -16,18 +17,18 @@ import org.slf4j.LoggerFactory;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = LoggerFactory
.getLogger(PdbFileIterator.class);
private final static Logger LOGGER = LoggerFactory.getLogger(PdbFileIterator.class);
private static final class EntrySupplier implements Supplier<Entry>,
AutoCloseable {
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles;
private PdbReader reader;
private PdbFile currentPdbFile;
private final Path storageBasePath;
public EntrySupplier(final Collection<PdbFile> pdbFiles) {
public EntrySupplier(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
super();
this.storageBasePath = storageBasePath;
this.pdbFiles = new ArrayDeque<>(pdbFiles);
}
@@ -73,14 +74,13 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
try {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile);
reader = new PdbReader(storageBasePath, currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.warn("the pdbFile " + currentPdbFile.getPath()
+ " is missing", e);
LOGGER.warn("the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
@@ -100,8 +100,8 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(pdbFiles);
public PdbFileIterator(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(storageBasePath, pdbFiles);
}
@Override

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java
View File

@@ -2,12 +2,14 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
class PdbFileUtils {
static OffsetDateTime dateOffset(final PdbFile pdbFile) throws FileNotFoundException, IOException {
static OffsetDateTime dateOffset(final Path storageBasePath, final PdbFile pdbFile)
throws FileNotFoundException, IOException {
try (PdbReader reader = new PdbReader(pdbFile)) {
try (PdbReader reader = new PdbReader(storageBasePath, pdbFile)) {
reader.seekToLastValue();
return reader.getDateOffsetAtCurrentPosition();
}

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
@@ -11,10 +12,11 @@ public class PdbFileViewer {
public static void main(final String[] args) throws FileNotFoundException, IOException {
final File file = new File(args[0]);
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path baseDirectory = file.toPath().getParent();
final PdbFile pdbFile = new PdbFile(file.toPath().getFileName(), TAGS);
long countMeasurements = 0;
try (final PdbReader reader = new PdbReader(pdbFile, false)) {
try (final PdbReader reader = new PdbReader(baseDirectory, pdbFile, false)) {
long value = 0;
int nextByte;

9
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
@@ -26,15 +27,15 @@ class PdbReader implements AutoCloseable {
private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws ReadException {
this(pdbFile, true);
public PdbReader(final Path storageBasePath, final PdbFile pdbFile) throws ReadException {
this(storageBasePath, pdbFile, true);
}
PdbReader(final PdbFile pdbFile, final boolean initialize) throws ReadException {
PdbReader(final Path storageBasePath, final PdbFile pdbFile, final boolean initialize) throws ReadException {
super();
try {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile));

18
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -6,6 +6,7 @@ import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import org.lucares.pdb.api.Entry;
@@ -80,15 +81,15 @@ class PdbWriter implements AutoCloseable, Flushable {
private final PdbFile pdbFile;
private long lastEpochMilli;
PdbWriter(final PdbFile pdbFile) throws IOException {
PdbWriter(final Path storageBasePath, final PdbFile pdbFile) throws IOException {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
if (storageFile.exists() && storageFile.length() > 0) {
// TODO @ahr check version
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(pdbFile);
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(storageBasePath, pdbFile);
lastEpochMilli = dateOffset.toInstant().toEpochMilli();
} else {
writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream);
@@ -105,7 +106,7 @@ class PdbWriter implements AutoCloseable, Flushable {
public OffsetDateTime getDateOffset() {
return DateUtils.epochMilliInUTC(lastEpochMilli);
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
@@ -167,16 +168,17 @@ class PdbWriter implements AutoCloseable, Flushable {
output.write(buffer, index, buffer.length - index);
}
public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException {
try (PdbWriter writer = new PdbWriter(pdbFile)) {
public static void writeEntry(final Path storageBasePath, final PdbFile pdbFile, final Entry... entries)
throws IOException {
try (PdbWriter writer = new PdbWriter(storageBasePath, pdbFile)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
}
public static void init(final PdbFile result) throws IOException {
writeEntry(result);
public static void init(final Path storageBasePath, final PdbFile result) throws IOException {
writeEntry(storageBasePath, result);
}
@Override

7
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -171,7 +171,7 @@ public class PerformanceDb implements AutoCloseable {
}
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final PdbFileIterator iterator = new PdbFileIterator(pdbFiles);
final PdbFileIterator iterator = new PdbFileIterator(db.getStorageBasePath(), pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
@@ -188,6 +188,11 @@ public class PerformanceDb implements AutoCloseable {
@Override
public void close() {
tagsToFile.close();
try {
db.close();
} catch (final IOException e) {
LOGGER.error("failed to close PdbDB", e);
}
}
public List<Proposal> autocomplete(final String query, final int caretIndex) {

6
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -72,7 +72,7 @@ public class TagsToFile implements AutoCloseable {
for (final Doc document : searchResult) {
final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver();
final Path path = document.getPath(resolver);
final Path path = document.getAbsolutePath(resolver);
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(path, tags);
@@ -160,7 +160,7 @@ public class TagsToFile implements AutoCloseable {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile);
final PdbWriter result = new PdbWriter(db.getStorageBasePath(), pdbFile);
getOrInit(tags).addWriter(result);
@@ -179,7 +179,7 @@ public class TagsToFile implements AutoCloseable {
final Path storageFile = db.createNewFile(tags);
final PdbFile result = new PdbFile(storageFile, tags);
PdbWriter.init(result);
PdbWriter.init(db.getStorageBasePath(), result);
return result;
}

14
performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java
View File

@@ -73,15 +73,16 @@ public class PdbReaderWriterTest {
public void testWriteRead(final List<Entry> entries) throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) {
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
try (final PdbReader reader = new PdbReader(pdbFile)) {
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
@@ -101,9 +102,10 @@ public class PdbReaderWriterTest {
final Entry entryA = new Entry(1, 1, TAGS);
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) {
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
writer.write(entryA);
}
@@ -116,7 +118,7 @@ public class PdbReaderWriterTest {
Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND);
try (final PdbReader reader = new PdbReader(pdbFile)) {
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actualA, entryA);

5
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -111,7 +111,8 @@ public class PerformanceDbTest {
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries);
final List<Path> filesInStorage = FileUtils.listRecursively(DataStore.storageDirectory(dataDirectory));
final Path storageBasePath = DataStore.storageDirectory(dataDirectory);
final List<Path> filesInStorage = FileUtils.listRecursively(storageBasePath);
Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv");
@@ -119,7 +120,7 @@ public class PerformanceDbTest {
final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags);
try (PdbReader pdbReader = new PdbReader(pdbFile)) {
try (PdbReader pdbReader = new PdbReader(storageBasePath, pdbFile)) {
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0));
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1));
Assert.assertEquals(pdbReader.readEntry().isPresent(), false);

17
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -31,8 +31,8 @@ public class TagsToFilesTest {
public void test() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue");
@@ -46,8 +46,9 @@ public class TagsToFilesTest {
}
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
@@ -66,8 +67,8 @@ public class TagsToFilesTest {
@Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -104,8 +105,8 @@ public class TagsToFilesTest {
public void testIdenticalDatesGoIntoSameFile() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);