From b61a34a0e63d6a4c3a08a3732c8ec549f7ca21a4 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 10 May 2018 17:41:50 +0200 Subject: [PATCH] use existing RandomAccessFile when updating the listing file Ingestion speed dropped drastically with the old implementation. In some situations to 7 entries per second over a 10 second period (sic!). When using the already opened RandomAccessFile the speed is back to previous values of 40k-50k entries per second on my 10 year old machine on an encrypted spinning disk. --- .../pdb/datastore/internal/FolderStorage.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java index c4abc1f..d19de2c 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java @@ -29,6 +29,7 @@ public class FolderStorage implements AutoCloseable { .getLogger("org.lucares.metrics.folderStorage.createListingFile"); private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory .getLogger("org.lucares.metrics.folderStorage.getPathByOffset"); + private final static Logger METRICS_INSERT = LoggerFactory.getLogger("org.lucares.metrics.folderStorage.insert"); private final Path storageBaseDirectory; @@ -49,7 +50,7 @@ public class FolderStorage implements AutoCloseable { this.maxFilesPerFolder = maxFilesPerFolder; init(); initListingFileIfNotExists(); - listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws"); + listingFile = new RandomAccessFile(listingFilePath.toFile(), "rw"); } @Override @@ -75,6 +76,7 @@ public class FolderStorage implements AutoCloseable { public ListingFileEntry insert(final String filenamePrefix, final String filenameSuffix) throws IOException { + final long start = System.nanoTime(); ensureCapacity(); String filename = filenamePrefix + "$" + filenameSuffix; @@ -88,6 +90,7 @@ public class FolderStorage implements AutoCloseable { filesInSecondLevel++; final ListingFileEntry result = updateListingFile(newFile); + METRICS_INSERT.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0); return result; } @@ -95,16 +98,13 @@ public class FolderStorage implements AutoCloseable { private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException { final long offsetInListingFile = Files.size(listingFilePath); // remember: all paths within storageBaseDirectory use only ascii characters - try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE, - StandardOpenOption.APPEND, StandardOpenOption.SYNC)) { - final Path relativePath = storageBaseDirectory.relativize(newFile); - listingFile.seek(offsetInListingFile); - listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII)); - listingFile.write(NEWLINE); - } - final String filename = newFile.getFileName().toString(); final Path relativePath = storageBaseDirectory.relativize(newFile); + listingFile.seek(offsetInListingFile); + listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII)); + listingFile.write(NEWLINE); + + final String filename = newFile.getFileName().toString(); return new ListingFileEntry(filename, offsetInListingFile, relativePath); } @@ -141,7 +141,7 @@ public class FolderStorage implements AutoCloseable { final long start = System.nanoTime(); LOGGER.info("listing file not found -> creating a new one"); createNewListingFile(); - METRICS_CREATE_LISTING_FILE.info(((System.nanoTime() - start) / 1_000_000.0) + "ms"); + METRICS_CREATE_LISTING_FILE.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0); } } @@ -180,7 +180,7 @@ public class FolderStorage implements AutoCloseable { } catch (final IOException e) { throw new RuntimeIOException(e); } finally { - METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms"); + METRICS_GET_PATH_BY_OFFSET.debug("{}ms", (System.nanoTime() - start) / 1_000_000.0); } }