andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use RandomAccessFile in FolderStorage.getPathByOffset()

Browse Source 
The old implementation opened a new buffered reader everytime
getPathByOffset was called. This took 1/20th of a second or
longer. For queries that visited thousands of files this could
take a long time.
We are now using a RandomAccessFile, that is opened once. The
average time spend in getPathByOffset is now down to 0.11ms.
This commit is contained in:
Andreas Huber
2018-05-10 10:22:25 +02:00
parent 82b8a8a932
commit 911062e26b
18 changed files with 215 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
@@ -16,18 +17,18 @@ import org.slf4j.LoggerFactory;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = LoggerFactory
.getLogger(PdbFileIterator.class);
private final static Logger LOGGER = LoggerFactory.getLogger(PdbFileIterator.class);
private static final class EntrySupplier implements Supplier<Entry>,
AutoCloseable {
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles;
private PdbReader reader;
private PdbFile currentPdbFile;
private final Path storageBasePath;
public EntrySupplier(final Collection<PdbFile> pdbFiles) {
public EntrySupplier(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
super();
this.storageBasePath = storageBasePath;
this.pdbFiles = new ArrayDeque<>(pdbFiles);
}
@@ -73,14 +74,13 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
try {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile);
reader = new PdbReader(storageBasePath, currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.warn("the pdbFile " + currentPdbFile.getPath()
+ " is missing", e);
LOGGER.warn("the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
@@ -100,8 +100,8 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(pdbFiles);
public PdbFileIterator(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(storageBasePath, pdbFiles);
}
@Override

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java
View File

@@ -2,12 +2,14 @@ package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
class PdbFileUtils {
static OffsetDateTime dateOffset(final PdbFile pdbFile) throws FileNotFoundException, IOException {
static OffsetDateTime dateOffset(final Path storageBasePath, final PdbFile pdbFile)
throws FileNotFoundException, IOException {
try (PdbReader reader = new PdbReader(pdbFile)) {
try (PdbReader reader = new PdbReader(storageBasePath, pdbFile)) {
reader.seekToLastValue();
return reader.getDateOffsetAtCurrentPosition();
}

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
@@ -11,10 +12,11 @@ public class PdbFileViewer {
public static void main(final String[] args) throws FileNotFoundException, IOException {
final File file = new File(args[0]);
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path baseDirectory = file.toPath().getParent();
final PdbFile pdbFile = new PdbFile(file.toPath().getFileName(), TAGS);
long countMeasurements = 0;
try (final PdbReader reader = new PdbReader(pdbFile, false)) {
try (final PdbReader reader = new PdbReader(baseDirectory, pdbFile, false)) {
long value = 0;
int nextByte;

9
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -6,6 +6,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
@@ -26,15 +27,15 @@ class PdbReader implements AutoCloseable {
private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws ReadException {
this(pdbFile, true);
public PdbReader(final Path storageBasePath, final PdbFile pdbFile) throws ReadException {
this(storageBasePath, pdbFile, true);
}
PdbReader(final PdbFile pdbFile, final boolean initialize) throws ReadException {
PdbReader(final Path storageBasePath, final PdbFile pdbFile, final boolean initialize) throws ReadException {
super();
try {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile));

18
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -6,6 +6,7 @@ import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import org.lucares.pdb.api.Entry;
@@ -80,15 +81,15 @@ class PdbWriter implements AutoCloseable, Flushable {
private final PdbFile pdbFile;
private long lastEpochMilli;
PdbWriter(final PdbFile pdbFile) throws IOException {
PdbWriter(final Path storageBasePath, final PdbFile pdbFile) throws IOException {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
if (storageFile.exists() && storageFile.length() > 0) {
// TODO @ahr check version
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(pdbFile);
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(storageBasePath, pdbFile);
lastEpochMilli = dateOffset.toInstant().toEpochMilli();
} else {
writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream);
@@ -105,7 +106,7 @@ class PdbWriter implements AutoCloseable, Flushable {
public OffsetDateTime getDateOffset() {
return DateUtils.epochMilliInUTC(lastEpochMilli);
}
public long getDateOffsetAsEpochMilli() {
return lastEpochMilli;
}
@@ -167,16 +168,17 @@ class PdbWriter implements AutoCloseable, Flushable {
output.write(buffer, index, buffer.length - index);
}
public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException {
try (PdbWriter writer = new PdbWriter(pdbFile)) {
public static void writeEntry(final Path storageBasePath, final PdbFile pdbFile, final Entry... entries)
throws IOException {
try (PdbWriter writer = new PdbWriter(storageBasePath, pdbFile)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
}
public static void init(final PdbFile result) throws IOException {
writeEntry(result);
public static void init(final Path storageBasePath, final PdbFile result) throws IOException {
writeEntry(storageBasePath, result);
}
@Override

7
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -171,7 +171,7 @@ public class PerformanceDb implements AutoCloseable {
}
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final PdbFileIterator iterator = new PdbFileIterator(pdbFiles);
final PdbFileIterator iterator = new PdbFileIterator(db.getStorageBasePath(), pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
@@ -188,6 +188,11 @@ public class PerformanceDb implements AutoCloseable {
@Override
public void close() {
tagsToFile.close();
try {
db.close();
} catch (final IOException e) {
LOGGER.error("failed to close PdbDB", e);
}
}
public List<Proposal> autocomplete(final String query, final int caretIndex) {

6
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -72,7 +72,7 @@ public class TagsToFile implements AutoCloseable {
for (final Doc document : searchResult) {
final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver();
final Path path = document.getPath(resolver);
final Path path = document.getAbsolutePath(resolver);
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(path, tags);
@@ -160,7 +160,7 @@ public class TagsToFile implements AutoCloseable {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile);
final PdbWriter result = new PdbWriter(db.getStorageBasePath(), pdbFile);
getOrInit(tags).addWriter(result);
@@ -179,7 +179,7 @@ public class TagsToFile implements AutoCloseable {
final Path storageFile = db.createNewFile(tags);
final PdbFile result = new PdbFile(storageFile, tags);
PdbWriter.init(result);
PdbWriter.init(db.getStorageBasePath(), result);
return result;
}

14
performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java
View File

@@ -73,15 +73,16 @@ public class PdbReaderWriterTest {
public void testWriteRead(final List<Entry> entries) throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) {
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
try (final PdbReader reader = new PdbReader(pdbFile)) {
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
@@ -101,9 +102,10 @@ public class PdbReaderWriterTest {
final Entry entryA = new Entry(1, 1, TAGS);
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(pdbFile)) {
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
writer.write(entryA);
}
@@ -116,7 +118,7 @@ public class PdbReaderWriterTest {
Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND);
try (final PdbReader reader = new PdbReader(pdbFile)) {
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actualA, entryA);

5
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -111,7 +111,8 @@ public class PerformanceDbTest {
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries);
final List<Path> filesInStorage = FileUtils.listRecursively(DataStore.storageDirectory(dataDirectory));
final Path storageBasePath = DataStore.storageDirectory(dataDirectory);
final List<Path> filesInStorage = FileUtils.listRecursively(storageBasePath);
Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv");
@@ -119,7 +120,7 @@ public class PerformanceDbTest {
final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags);
try (PdbReader pdbReader = new PdbReader(pdbFile)) {
try (PdbReader pdbReader = new PdbReader(storageBasePath, pdbFile)) {
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0));
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1));
Assert.assertEquals(pdbReader.readEntry().isPresent(), false);

17
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -31,8 +31,8 @@ public class TagsToFilesTest {
public void test() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue");
@@ -46,8 +46,9 @@ public class TagsToFilesTest {
}
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
@@ -66,8 +67,8 @@ public class TagsToFilesTest {
@Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -104,8 +105,8 @@ public class TagsToFilesTest {
public void testIdenticalDatesGoIntoSameFile() throws Exception {
final PdbDB db = new PdbDB(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
try (final PdbDB db = new PdbDB(dataDirectory); //
final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);