andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use existing RandomAccessFile when updating the listing file

Browse Source 
Ingestion speed dropped drastically with the old implementation.
In some situations to 7 entries per second over a 10 second period
(sic!). When using the already opened RandomAccessFile the speed
is back to previous values of 40k-50k entries per second on my 10 year
old machine on an encrypted spinning disk.
This commit is contained in:
Andreas Huber
2018-05-10 17:41:50 +02:00
parent 911062e26b
commit b61a34a0e6
1 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java
View File

@@ -29,6 +29,7 @@ public class FolderStorage implements AutoCloseable {
.getLogger("org.lucares.metrics.folderStorage.createListingFile");
private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.getPathByOffset");
private final static Logger METRICS_INSERT = LoggerFactory.getLogger("org.lucares.metrics.folderStorage.insert");
private final Path storageBaseDirectory;
@@ -49,7 +50,7 @@ public class FolderStorage implements AutoCloseable {
this.maxFilesPerFolder = maxFilesPerFolder;
init();
initListingFileIfNotExists();
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws");
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rw");
}
@Override
@@ -75,6 +76,7 @@ public class FolderStorage implements AutoCloseable {
public ListingFileEntry insert(final String filenamePrefix, final String filenameSuffix) throws IOException {
final long start = System.nanoTime();
ensureCapacity();
String filename = filenamePrefix + "$" + filenameSuffix;
@@ -88,6 +90,7 @@ public class FolderStorage implements AutoCloseable {
filesInSecondLevel++;
final ListingFileEntry result = updateListingFile(newFile);
METRICS_INSERT.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
return result;
}
@@ -95,16 +98,13 @@ public class FolderStorage implements AutoCloseable {
private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException {
final long offsetInListingFile = Files.size(listingFilePath);
// remember: all paths within storageBaseDirectory use only ascii characters
try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE,
StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
final Path relativePath = storageBaseDirectory.relativize(newFile);
listingFile.seek(offsetInListingFile);
listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE);
}
final String filename = newFile.getFileName().toString();
final Path relativePath = storageBaseDirectory.relativize(newFile);
return new ListingFileEntry(filename, offsetInListingFile, relativePath);
}
@@ -141,7 +141,7 @@ public class FolderStorage implements AutoCloseable {
final long start = System.nanoTime();
LOGGER.info("listing file not found -> creating a new one");
createNewListingFile();
METRICS_CREATE_LISTING_FILE.info(((System.nanoTime() - start) / 1_000_000.0) + "ms");
METRICS_CREATE_LISTING_FILE.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
}
}
@@ -180,7 +180,7 @@ public class FolderStorage implements AutoCloseable {
} catch (final IOException e) {
throw new RuntimeIOException(e);
} finally {
METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms");
METRICS_GET_PATH_BY_OFFSET.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
}
}