andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

use existing RandomAccessFile when updating the listing file

Browse Source 
Ingestion speed dropped drastically with the old implementation.
In some situations to 7 entries per second over a 10 second period
(sic!). When using the already opened RandomAccessFile the speed
is back to previous values of 40k-50k entries per second on my 10 year
old machine on an encrypted spinning disk.
This commit is contained in:
Andreas Huber
2018-05-10 17:41:50 +02:00
parent 911062e26b
commit b61a34a0e6
1 changed files with 11 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java
View File

@@ -29,6 +29,7 @@ public class FolderStorage implements AutoCloseable {
.getLogger("org.lucares.metrics.folderStorage.createListingFile"); .getLogger("org.lucares.metrics.folderStorage.createListingFile");
private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory
.getLogger("org.lucares.metrics.folderStorage.getPathByOffset"); .getLogger("org.lucares.metrics.folderStorage.getPathByOffset");
private final static Logger METRICS_INSERT = LoggerFactory.getLogger("org.lucares.metrics.folderStorage.insert");
private final Path storageBaseDirectory; private final Path storageBaseDirectory;
@@ -49,7 +50,7 @@ public class FolderStorage implements AutoCloseable {
this.maxFilesPerFolder = maxFilesPerFolder; this.maxFilesPerFolder = maxFilesPerFolder;
init(); init();
initListingFileIfNotExists(); initListingFileIfNotExists();
listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws"); listingFile = new RandomAccessFile(listingFilePath.toFile(), "rw");
} }
@Override @Override
@@ -75,6 +76,7 @@ public class FolderStorage implements AutoCloseable {
public ListingFileEntry insert(final String filenamePrefix, final String filenameSuffix) throws IOException { public ListingFileEntry insert(final String filenamePrefix, final String filenameSuffix) throws IOException {
final long start = System.nanoTime();
ensureCapacity(); ensureCapacity();
String filename = filenamePrefix + "$" + filenameSuffix; String filename = filenamePrefix + "$" + filenameSuffix;
@@ -88,6 +90,7 @@ public class FolderStorage implements AutoCloseable {
filesInSecondLevel++; filesInSecondLevel++;
final ListingFileEntry result = updateListingFile(newFile); final ListingFileEntry result = updateListingFile(newFile);
METRICS_INSERT.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
return result; return result;
} }
@@ -95,16 +98,13 @@ public class FolderStorage implements AutoCloseable {
private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException { private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException {
final long offsetInListingFile = Files.size(listingFilePath); final long offsetInListingFile = Files.size(listingFilePath);
// remember: all paths within storageBaseDirectory use only ascii characters // remember: all paths within storageBaseDirectory use only ascii characters
try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE,
StandardOpenOption.APPEND, StandardOpenOption.SYNC)) {
final Path relativePath = storageBaseDirectory.relativize(newFile); final Path relativePath = storageBaseDirectory.relativize(newFile);
listingFile.seek(offsetInListingFile); listingFile.seek(offsetInListingFile);
listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII)); listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII));
listingFile.write(NEWLINE); listingFile.write(NEWLINE);
}
final String filename = newFile.getFileName().toString(); final String filename = newFile.getFileName().toString();
final Path relativePath = storageBaseDirectory.relativize(newFile);
return new ListingFileEntry(filename, offsetInListingFile, relativePath); return new ListingFileEntry(filename, offsetInListingFile, relativePath);
} }
@@ -141,7 +141,7 @@ public class FolderStorage implements AutoCloseable {
final long start = System.nanoTime(); final long start = System.nanoTime();
LOGGER.info("listing file not found -> creating a new one"); LOGGER.info("listing file not found -> creating a new one");
createNewListingFile(); createNewListingFile();
METRICS_CREATE_LISTING_FILE.info(((System.nanoTime() - start) / 1_000_000.0) + "ms"); METRICS_CREATE_LISTING_FILE.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
} }
} }
@@ -180,7 +180,7 @@ public class FolderStorage implements AutoCloseable {
} catch (final IOException e) { } catch (final IOException e) {
throw new RuntimeIOException(e); throw new RuntimeIOException(e);
} finally { } finally {
METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms"); METRICS_GET_PATH_BY_OFFSET.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0);
} }
} }