From 911062e26b51fe81a201d9a2de49f4d29fcb02fa Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 10 May 2018 10:22:25 +0200 Subject: [PATCH] use RandomAccessFile in FolderStorage.getPathByOffset() The old implementation opened a new buffered reader everytime getPathByOffset was called. This took 1/20th of a second or longer. For queries that visited thousands of files this could take a long time. We are now using a RandomAccessFile, that is opened once. The average time spend in getPathByOffset is now down to 0.11ms. --- .../java/org/lucares/pdb/datastore/Doc.java | 29 ++++--- .../java/org/lucares/pdb/datastore/PdbDB.java | 11 ++- .../pdb/datastore/internal/DataStore.java | 20 ++++- .../pdb/datastore/internal/FolderStorage.java | 87 ++++++++++--------- .../datastore/internal/ListingFileEntry.java | 38 +++++--- .../pdb/datastore/internal/DataStoreTest.java | 14 +-- .../datastore/internal/FolderStorageTest.java | 53 +++++------ .../pdb/datastore/internal/ProposerTest.java | 1 + .../performance/db/PdbFileIterator.java | 20 ++--- .../lucares/performance/db/PdbFileUtils.java | 6 +- .../lucares/performance/db/PdbFileViewer.java | 6 +- .../org/lucares/performance/db/PdbReader.java | 9 +- .../org/lucares/performance/db/PdbWriter.java | 18 ++-- .../lucares/performance/db/PerformanceDb.java | 7 +- .../lucares/performance/db/TagsToFile.java | 6 +- .../performance/db/PdbReaderWriterTest.java | 14 +-- .../performance/db/PerformanceDbTest.java | 5 +- .../performance/db/TagsToFilesTest.java | 17 ++-- 18 files changed, 215 insertions(+), 146 deletions(-) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/Doc.java b/data-store/src/main/java/org/lucares/pdb/datastore/Doc.java index 1536d69..29daebc 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/Doc.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/Doc.java @@ -10,6 +10,7 @@ import org.lucares.pdb.datastore.internal.DataStore; public class Doc { private final Tags tags; private final long offsetInListingFile; + private final Path storageBasePath; private byte[] path; /** @@ -24,21 +25,25 @@ public class Doc { * @param tags * @param offsetInListingFile * must be set if {@code path} is {@code null} - * @param path - * optional, can be {@code null} + * @param storageBasePath + * the storage base path. + * @param relativePath + * optional, can be {@code null}. This path is relative to + * {@code storageBasePath} */ - public Doc(final Tags tags, final long offsetInListingFile, final Path path) { + public Doc(final Tags tags, final long offsetInListingFile, final Path storageBasePath, final Path relativePath) { super(); this.tags = tags; this.offsetInListingFile = offsetInListingFile; - setPath(path); + this.storageBasePath = storageBasePath; + setRelativePath(relativePath); } public Tags getTags() { return tags; } - public void setPath(final Path path) { + public void setRelativePath(final Path path) { if (path != null) { this.path = path.toString().getBytes(StandardCharsets.UTF_8); } else { @@ -54,18 +59,18 @@ public class Doc { * * @return the path */ - public Path getPath(final FolderStoragePathResolver resolver) { + public Path getAbsolutePath(final FolderStoragePathResolver resolver) { if (path == null) { final Path resolvedPath = resolver.getPath(offsetInListingFile); - setPath(resolvedPath); + setRelativePath(resolvedPath); } - final Path result = Paths.get(new String(path, StandardCharsets.UTF_8)); - return result; + final Path relativePath = Paths.get(new String(path, StandardCharsets.UTF_8)); + return storageBasePath.resolve(relativePath); } - private Path getPathNullable() { - return getPath(FolderStoragePathResolver.NULL); + private Path getAbsolutePathNullable() { + return getAbsolutePath(FolderStoragePathResolver.NULL); } public long getOffsetInListingFile() { @@ -74,7 +79,7 @@ public class Doc { @Override public String toString() { - return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getPathNullable() + return "Doc [tags=" + tags + ", offsetInListingFile=" + offsetInListingFile + ", path=" + getAbsolutePathNullable() + "]"; } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java b/data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java index 93c2f3a..8c5505b 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/PdbDB.java @@ -9,7 +9,7 @@ import org.lucares.pdb.api.Tags; import org.lucares.pdb.datastore.internal.DataStore; import org.lucares.pdb.datastore.internal.Proposer; -public class PdbDB { +public class PdbDB implements AutoCloseable { private final DataStore dataStore; private final Proposer proposer; @@ -47,4 +47,13 @@ public class PdbDB { public FolderStoragePathResolver getFolderStoragePathResolver() { return dataStore.getFolderStoragePathResolver(); } + + public Path getStorageBasePath() { + return dataStore.getStorageBasePath(); + } + + @Override + public void close() throws IOException { + dataStore.close(); + } } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java index d1afa04..c60972d 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java @@ -25,7 +25,7 @@ import org.lucares.pdb.datastore.lang.QueryLanguageParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DataStore { +public class DataStore implements AutoCloseable { private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory .getLogger("org.lucares.metrics.dataStore.executeQuery"); private static final Logger INITIALIZE = LoggerFactory.getLogger("org.lucares.metrics.dataStore.init"); @@ -43,11 +43,13 @@ public class DataStore { private final FolderStorage folderStorage; private final FolderStoragePathResolver folderStoragePathResolver; + private final Path storageBasePath; public DataStore(final Path dataDirectory) throws IOException { Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(dataDirectory)); - folderStorage = new FolderStorage(storageDirectory(dataDirectory), 1000); + storageBasePath = storageDirectory(dataDirectory); + folderStorage = new FolderStorage(storageBasePath, 1000); init(folderStorage); folderStoragePathResolver = folderStorage::getPathByOffset; @@ -60,7 +62,7 @@ public class DataStore { files// .parallel() .forEach(listingFileEntry -> { - listingFileEntry.unsetPath(); // unset the path, so that we don't store it for every document (will + listingFileEntry.unsetRelativePath(); // unset the path, so that we don't store it for every document (will // be // initialized lazily if needed) @@ -80,7 +82,8 @@ public class DataStore { private void cacheTagToFileMapping(final Tags tags, final ListingFileEntry listingFileEntry) { final int docId; - final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), listingFileEntry.getPath()); + final Doc newDoc = new Doc(tags, listingFileEntry.getOffsetInListingFile(), storageBasePath, + listingFileEntry.getPath()); synchronized (docIdToDoc) { docId = docIdToDoc.size(); docIdToDoc.add(newDoc); @@ -245,4 +248,13 @@ public class DataStore { public FolderStoragePathResolver getFolderStoragePathResolver() { return folderStoragePathResolver; } + + public Path getStorageBasePath() { + return storageBasePath; + } + + @Override + public void close() throws IOException { + folderStorage.close(); + } } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java index 773d404..c4abc1f 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/FolderStorage.java @@ -1,7 +1,5 @@ package org.lucares.pdb.datastore.internal; -import java.io.BufferedReader; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.io.Writer; @@ -22,12 +20,15 @@ import org.lucares.pdb.api.RuntimeIOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FolderStorage { +public class FolderStorage implements AutoCloseable { + private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.US_ASCII); static final String LISTING_FILE_NAME = "listing.csv"; private final static Logger LOGGER = LoggerFactory.getLogger(FolderStorage.class); private final static Logger METRICS_CREATE_LISTING_FILE = LoggerFactory - .getLogger("org.lucares.metrics.fodlerStorage.createListingFile"); + .getLogger("org.lucares.metrics.folderStorage.createListingFile"); + private final static Logger METRICS_GET_PATH_BY_OFFSET = LoggerFactory + .getLogger("org.lucares.metrics.folderStorage.getPathByOffset"); private final Path storageBaseDirectory; @@ -39,14 +40,21 @@ public class FolderStorage { private final int maxFilesPerFolder; - private final Path listingFile; + private final Path listingFilePath; + private final RandomAccessFile listingFile; public FolderStorage(final Path storageBaseDirectory, final int maxFilesPerFolder) throws IOException { this.storageBaseDirectory = storageBaseDirectory; - this.listingFile = storageBaseDirectory.resolve(LISTING_FILE_NAME); + this.listingFilePath = storageBaseDirectory.resolve(LISTING_FILE_NAME); this.maxFilesPerFolder = maxFilesPerFolder; init(); initListingFileIfNotExists(); + listingFile = new RandomAccessFile(listingFilePath.toFile(), "rws"); + } + + @Override + public void close() throws IOException { + listingFile.close(); } private void init() throws IOException { @@ -85,23 +93,19 @@ public class FolderStorage { } private synchronized ListingFileEntry updateListingFile(final Path newFile) throws IOException { - final long offsetInListingFile = getFilePointer(); - try (Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE, - StandardOpenOption.APPEND)) { - out.write(newFile.toString()); - out.write("\n"); + final long offsetInListingFile = Files.size(listingFilePath); + // remember: all paths within storageBaseDirectory use only ascii characters + try (Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, StandardOpenOption.CREATE, + StandardOpenOption.APPEND, StandardOpenOption.SYNC)) { + + final Path relativePath = storageBaseDirectory.relativize(newFile); + listingFile.seek(offsetInListingFile); + listingFile.write(relativePath.toString().getBytes(StandardCharsets.US_ASCII)); + listingFile.write(NEWLINE); } final String filename = newFile.getFileName().toString(); - return new ListingFileEntry(filename, offsetInListingFile, newFile); - } - - private long getFilePointer() throws FileNotFoundException, IOException { - final RandomAccessFile randomAccessFile = new RandomAccessFile(listingFile.toFile(), "r"); - try { - return randomAccessFile.getFilePointer(); - } finally { - randomAccessFile.close(); - } + final Path relativePath = storageBaseDirectory.relativize(newFile); + return new ListingFileEntry(filename, offsetInListingFile, relativePath); } private void ensureCapacity() throws IOException { @@ -125,21 +129,15 @@ public class FolderStorage { public Stream list() throws IOException { - return readListingFile(); - } - - private Stream readListingFile() throws IOException { - - try (final ListingFileIterator iterator = new ListingFileIterator(listingFile)) { - final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, - Spliterator.ORDERED); - final Stream stream = StreamSupport.stream(spliterator, false); - return stream; - } + final ListingFileIterator iterator = new ListingFileIterator(listingFilePath); + final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, + Spliterator.ORDERED); + final Stream stream = StreamSupport.stream(spliterator, false); + return stream; } private void initListingFileIfNotExists() throws IOException { - if (!Files.exists(listingFile)) { + if (!Files.exists(listingFilePath)) { final long start = System.nanoTime(); LOGGER.info("listing file not found -> creating a new one"); createNewListingFile(); @@ -151,29 +149,38 @@ public class FolderStorage { final int maxDepth = Integer.MAX_VALUE; final BiPredicate matchRegularFiles = (path, attr) -> Files.isRegularFile(path); - try (final Writer out = Files.newBufferedWriter(listingFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE, - StandardOpenOption.APPEND); + // remember: all paths within storageBaseDirectory use only ascii characters + try (final Writer out = Files.newBufferedWriter(listingFilePath, StandardCharsets.US_ASCII, + StandardOpenOption.CREATE, StandardOpenOption.APPEND); final Stream stream = Files.find(storageBaseDirectory, maxDepth, matchRegularFiles)) { final Iterator iterator = stream.iterator(); while (iterator.hasNext()) { final Path path = iterator.next(); if (!path.getFileName().toString().equals(LISTING_FILE_NAME)) { - out.write(path.toString()); + + final Path relativePath = storageBaseDirectory.relativize(path); + + out.write(relativePath.toString()); out.write("\n"); } } } } - public Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException { + public synchronized Path getPathByOffset(final long offsetInListingFile) throws RuntimeIOException { - try (BufferedReader reader = Files.newBufferedReader(listingFile, StandardCharsets.UTF_8)) { - reader.skip(offsetInListingFile); - final String line = reader.readLine(); + final long start = System.nanoTime(); + try { + listingFile.seek(offsetInListingFile); + + // remember: all paths within storageBaseDirectory use only ascii characters + final String line = listingFile.readLine(); return Paths.get(line); } catch (final IOException e) { throw new RuntimeIOException(e); + } finally { + METRICS_GET_PATH_BY_OFFSET.debug(((System.nanoTime() - start) / 1_000_000.0) + "ms"); } } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java index b0a40a3..89e2868 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ListingFileEntry.java @@ -4,15 +4,29 @@ import java.nio.file.Path; import javax.annotation.Nullable; +import org.lucares.pdb.datastore.Doc; + public class ListingFileEntry { private final String filename; private final long offsetInListingFile; - private Path path; + private Path relativePath; - public ListingFileEntry(final String filename, final long offsetInListingFile, final Path path) { + /** + * Create a new {@link ListingFileEntry}. + *

+ * The {@code path} is optional. When the {@link ListingFileEntry} is read from + * the listing file, then the {@code path} is set to {@code null}. This is done + * to save memory. See {@link Doc} for more information on its usage. + * + * @param filename + * @param offsetInListingFile + * @param relativePath + * optional, see {@link Doc} + */ + public ListingFileEntry(final String filename, final long offsetInListingFile, final Path relativePath) { this.filename = filename; this.offsetInListingFile = offsetInListingFile; - this.path = path; + this.relativePath = relativePath; } public String getFilename() { @@ -23,19 +37,19 @@ public class ListingFileEntry { return offsetInListingFile; } - public void unsetPath() { - path = null; + public void unsetRelativePath() { + relativePath = null; } @Nullable public Path getPath() { - return path; + return relativePath; } @Override public String toString() { - return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile + ", path=" - + path + "]"; + return "ListingFileEntry [filename=" + filename + ", offsetInListingFile=" + offsetInListingFile + + ", relativePath=" + relativePath + "]"; } @Override @@ -44,7 +58,7 @@ public class ListingFileEntry { int result = 1; result = prime * result + ((filename == null) ? 0 : filename.hashCode()); result = prime * result + (int) (offsetInListingFile ^ (offsetInListingFile >>> 32)); - result = prime * result + ((path == null) ? 0 : path.hashCode()); + result = prime * result + ((relativePath == null) ? 0 : relativePath.hashCode()); return result; } @@ -64,10 +78,10 @@ public class ListingFileEntry { return false; if (offsetInListingFile != other.offsetInListingFile) return false; - if (path == null) { - if (other.path != null) + if (relativePath == null) { + if (other.relativePath != null) return false; - } else if (!path.equals(other.path)) + } else if (!relativePath.equals(other.relativePath)) return false; return true; } diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java index 81b23e2..76b7e38 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java @@ -42,14 +42,16 @@ public class DataStoreTest { final Path path; { final DataStore dataStore = new DataStore(dataDirectory); + final Path storageBasePath = dataStore.getStorageBasePath(); final Tags tags = Tags.create("key1", "value1", "key2", "value2"); path = dataStore.createNewFile(tags); - assertSearch(dataStore, "key1=value1", path); + assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path)); } { final DataStore dataStore = new DataStore(dataDirectory); - assertSearch(dataStore, "key1=value1", path); + final Path storageBasePath = dataStore.getStorageBasePath(); + assertSearch(dataStore, "key1=value1", storageBasePath.resolve(path)); } } @@ -124,9 +126,11 @@ public class DataStoreTest { private void assertSearch(final String query, final Tags... tags) { final List actualDocs = dataStore.search(query); final List actual = CollectionUtils.map(actualDocs, - doc -> doc.getPath(dataStore.getFolderStoragePathResolver())); + doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver())); - final List expectedPaths = CollectionUtils.map(tags, tagsToPath::get); + final Path storageBasePath = dataStore.getStorageBasePath(); + final List expectedPaths = CollectionUtils.map(CollectionUtils.map(tags, tagsToPath::get), + storageBasePath::resolve); Assert.assertEquals(actual, expectedPaths, "Query: " + query + " Found: " + getTagsForPaths(actual)); } @@ -154,7 +158,7 @@ public class DataStoreTest { private void assertSearch(final DataStore dataStore, final String query, final Path... paths) { final List actualDocs = dataStore.search(query); final List actual = CollectionUtils.map(actualDocs, - doc -> doc.getPath(dataStore.getFolderStoragePathResolver())); + doc -> doc.getAbsolutePath(dataStore.getFolderStoragePathResolver())); Assert.assertEquals(actual, Arrays.asList(paths)); } diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java index b9e79d4..79a7156 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/FolderStorageTest.java @@ -79,40 +79,40 @@ public class FolderStorageTest { @Test public void testCreateAndUpdateFileListing() throws Exception { final int maxFilesPerFolder = 10; - final Path storageLeafFolder = dataDirectory.resolve("0").resolve("0"); - final int storageLeafFolderLength = storageLeafFolder.toString().length(); // initial creation { - final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); - storage.insert("abc", ".txt"); - storage.insert("def", ".txt"); + try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) { + storage.insert("abc", ".txt"); + storage.insert("def", ".txt"); - final List initialListing = storage.list().collect(Collectors.toList()); - Assert.assertEquals(initialListing, Arrays.asList(// - new ListingFileEntry("abc$.txt", 0, null), // - new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null))); + final List initialListing = storage.list().collect(Collectors.toList()); + Assert.assertEquals(initialListing, Arrays.asList(// + new ListingFileEntry("abc$.txt", 0, null), // + new ListingFileEntry("def$.txt", 13, null))); + } } // load existing storage { - final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); + try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder);) { - // files inserted previously are still there - final List initialListing = storage.list().collect(Collectors.toList()); + // files inserted previously are still there + final List initialListing = storage.list().collect(Collectors.toList()); - Assert.assertEquals(initialListing, Arrays.asList(// - new ListingFileEntry("abc$.txt", 0, null), // - new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null))); + Assert.assertEquals(initialListing, Arrays.asList(// + new ListingFileEntry("abc$.txt", 0, null), // + new ListingFileEntry("def$.txt", 13, null))); - // add new file - storage.insert("ghi", ".txt"); + // add new file + storage.insert("ghi", ".txt"); - // listing is updated - final List updatedListing = storage.list().collect(Collectors.toList()); - Assert.assertEquals(updatedListing, Arrays.asList(// - new ListingFileEntry("abc$.txt", 0, null), // - new ListingFileEntry("def$.txt", storageLeafFolderLength + 10, null), // - new ListingFileEntry("ghi$.txt", 2 * storageLeafFolderLength + 20, null))); + // listing is updated + final List updatedListing = storage.list().collect(Collectors.toList()); + Assert.assertEquals(updatedListing, Arrays.asList(// + new ListingFileEntry("abc$.txt", 0, null), // + new ListingFileEntry("def$.txt", 13, null), // + new ListingFileEntry("ghi$.txt", 26, null))); + } } } @@ -127,10 +127,11 @@ public class FolderStorageTest { } private void storeFiles(final int maxFilesPerFolder, final String... filenames) throws IOException { - final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder); + try (final FolderStorage storage = new FolderStorage(dataDirectory, maxFilesPerFolder)) { - for (final String filename : filenames) { - storage.insert(filename, SUFFIX); + for (final String filename : filenames) { + storage.insert(filename, SUFFIX); + } } } } diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java index c45f2de..5bc37a8 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java @@ -34,6 +34,7 @@ public class ProposerTest { @AfterClass public void afterClass() throws IOException { FileUtils.delete(dataDirectory); + db.close(); db = null; tagsToPath = null; Tags.STRING_COMPRESSOR = null; diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index ab08ccd..6571df2 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -3,6 +3,7 @@ package org.lucares.performance.db; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayDeque; import java.util.Collection; import java.util.Iterator; @@ -16,18 +17,18 @@ import org.slf4j.LoggerFactory; public class PdbFileIterator implements Iterator, AutoCloseable { - private final static Logger LOGGER = LoggerFactory - .getLogger(PdbFileIterator.class); + private final static Logger LOGGER = LoggerFactory.getLogger(PdbFileIterator.class); - private static final class EntrySupplier implements Supplier, - AutoCloseable { + private static final class EntrySupplier implements Supplier, AutoCloseable { private final Queue pdbFiles; private PdbReader reader; private PdbFile currentPdbFile; + private final Path storageBasePath; - public EntrySupplier(final Collection pdbFiles) { + public EntrySupplier(final Path storageBasePath, final Collection pdbFiles) { super(); + this.storageBasePath = storageBasePath; this.pdbFiles = new ArrayDeque<>(pdbFiles); } @@ -73,14 +74,13 @@ public class PdbFileIterator implements Iterator, AutoCloseable { try { if (Files.size(currentPdbFile.getPath()) > 0) { - reader = new PdbReader(currentPdbFile); + reader = new PdbReader(storageBasePath, currentPdbFile); break; } else { LOGGER.info("ignoring empty file " + currentPdbFile); } } catch (final FileNotFoundException e) { - LOGGER.warn("the pdbFile " + currentPdbFile.getPath() - + " is missing", e); + LOGGER.warn("the pdbFile " + currentPdbFile.getPath() + " is missing", e); } catch (final IOException e) { throw new ReadException(e); } @@ -100,8 +100,8 @@ public class PdbFileIterator implements Iterator, AutoCloseable { private Optional next = Optional.empty(); - public PdbFileIterator(final Collection pdbFiles) { - supplier = new EntrySupplier(pdbFiles); + public PdbFileIterator(final Path storageBasePath, final Collection pdbFiles) { + supplier = new EntrySupplier(storageBasePath, pdbFiles); } @Override diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java index de621fb..5b3626b 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java @@ -2,12 +2,14 @@ package org.lucares.performance.db; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.time.OffsetDateTime; class PdbFileUtils { - static OffsetDateTime dateOffset(final PdbFile pdbFile) throws FileNotFoundException, IOException { + static OffsetDateTime dateOffset(final Path storageBasePath, final PdbFile pdbFile) + throws FileNotFoundException, IOException { - try (PdbReader reader = new PdbReader(pdbFile)) { + try (PdbReader reader = new PdbReader(storageBasePath, pdbFile)) { reader.seekToLastValue(); return reader.getDateOffsetAtCurrentPosition(); } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java index 51e5abe..eaefe37 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java @@ -3,6 +3,7 @@ package org.lucares.performance.db; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import org.lucares.pdb.api.Tags; @@ -11,10 +12,11 @@ public class PdbFileViewer { public static void main(final String[] args) throws FileNotFoundException, IOException { final File file = new File(args[0]); - final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); + final Path baseDirectory = file.toPath().getParent(); + final PdbFile pdbFile = new PdbFile(file.toPath().getFileName(), TAGS); long countMeasurements = 0; - try (final PdbReader reader = new PdbReader(pdbFile, false)) { + try (final PdbReader reader = new PdbReader(baseDirectory, pdbFile, false)) { long value = 0; int nextByte; diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index 3f51deb..bed6fbb 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -6,6 +6,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; @@ -26,15 +27,15 @@ class PdbReader implements AutoCloseable { private final PdbFile pdbFile; - public PdbReader(final PdbFile pdbFile) throws ReadException { - this(pdbFile, true); + public PdbReader(final Path storageBasePath, final PdbFile pdbFile) throws ReadException { + this(storageBasePath, pdbFile, true); } - PdbReader(final PdbFile pdbFile, final boolean initialize) throws ReadException { + PdbReader(final Path storageBasePath, final PdbFile pdbFile, final boolean initialize) throws ReadException { super(); try { this.pdbFile = pdbFile; - final File storageFile = pdbFile.getPath().toFile(); + final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile(); this.data = new BufferedInputStream(new FileInputStream(storageFile)); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java index fcd1fcf..ad1063d 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -6,6 +6,7 @@ import java.io.FileOutputStream; import java.io.Flushable; import java.io.IOException; import java.io.OutputStream; +import java.nio.file.Path; import java.time.OffsetDateTime; import org.lucares.pdb.api.Entry; @@ -80,15 +81,15 @@ class PdbWriter implements AutoCloseable, Flushable { private final PdbFile pdbFile; private long lastEpochMilli; - PdbWriter(final PdbFile pdbFile) throws IOException { + PdbWriter(final Path storageBasePath, final PdbFile pdbFile) throws IOException { this.pdbFile = pdbFile; - final File storageFile = pdbFile.getPath().toFile(); + final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile(); this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND)); if (storageFile.exists() && storageFile.length() > 0) { // TODO @ahr check version - final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(pdbFile); + final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(storageBasePath, pdbFile); lastEpochMilli = dateOffset.toInstant().toEpochMilli(); } else { writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream); @@ -105,7 +106,7 @@ class PdbWriter implements AutoCloseable, Flushable { public OffsetDateTime getDateOffset() { return DateUtils.epochMilliInUTC(lastEpochMilli); } - + public long getDateOffsetAsEpochMilli() { return lastEpochMilli; } @@ -167,16 +168,17 @@ class PdbWriter implements AutoCloseable, Flushable { output.write(buffer, index, buffer.length - index); } - public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException { - try (PdbWriter writer = new PdbWriter(pdbFile)) { + public static void writeEntry(final Path storageBasePath, final PdbFile pdbFile, final Entry... entries) + throws IOException { + try (PdbWriter writer = new PdbWriter(storageBasePath, pdbFile)) { for (final Entry entry : entries) { writer.write(entry); } } } - public static void init(final PdbFile result) throws IOException { - writeEntry(result); + public static void init(final Path storageBasePath, final PdbFile result) throws IOException { + writeEntry(storageBasePath, result); } @Override diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index fa53c11..7de050e 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -171,7 +171,7 @@ public class PerformanceDb implements AutoCloseable { } private Stream toStream(final List pdbFiles) { - final PdbFileIterator iterator = new PdbFileIterator(pdbFiles); + final PdbFileIterator iterator = new PdbFileIterator(db.getStorageBasePath(), pdbFiles); final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); final Stream stream = StreamSupport.stream(spliterator, false); @@ -188,6 +188,11 @@ public class PerformanceDb implements AutoCloseable { @Override public void close() { tagsToFile.close(); + try { + db.close(); + } catch (final IOException e) { + LOGGER.error("failed to close PdbDB", e); + } } public List autocomplete(final String query, final int caretIndex) { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index 77f1749..064cb37 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -72,7 +72,7 @@ public class TagsToFile implements AutoCloseable { for (final Doc document : searchResult) { final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver(); - final Path path = document.getPath(resolver); + final Path path = document.getAbsolutePath(resolver); final Tags tags = document.getTags(); final PdbFile pdbFile = new PdbFile(path, tags); @@ -160,7 +160,7 @@ public class TagsToFile implements AutoCloseable { final long start = System.nanoTime(); try { final PdbFile pdbFile = createNewPdbFile(tags); - final PdbWriter result = new PdbWriter(pdbFile); + final PdbWriter result = new PdbWriter(db.getStorageBasePath(), pdbFile); getOrInit(tags).addWriter(result); @@ -179,7 +179,7 @@ public class TagsToFile implements AutoCloseable { final Path storageFile = db.createNewFile(tags); final PdbFile result = new PdbFile(storageFile, tags); - PdbWriter.init(result); + PdbWriter.init(db.getStorageBasePath(), result); return result; } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java index f431255..54f034b 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java @@ -73,15 +73,16 @@ public class PdbReaderWriterTest { public void testWriteRead(final List entries) throws Exception { final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); - final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); + final Path relativePath = dataDirectory.relativize(file.toPath()); + final PdbFile pdbFile = new PdbFile(relativePath, TAGS); - try (PdbWriter writer = new PdbWriter(pdbFile)) { + try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) { for (final Entry entry : entries) { writer.write(entry); } } - try (final PdbReader reader = new PdbReader(pdbFile)) { + try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) { for (final Entry entry : entries) { @@ -101,9 +102,10 @@ public class PdbReaderWriterTest { final Entry entryA = new Entry(1, 1, TAGS); final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); - final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS); + final Path relativePath = dataDirectory.relativize(file.toPath()); + final PdbFile pdbFile = new PdbFile(relativePath, TAGS); - try (PdbWriter writer = new PdbWriter(pdbFile)) { + try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) { writer.write(entryA); } @@ -116,7 +118,7 @@ public class PdbReaderWriterTest { Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND); - try (final PdbReader reader = new PdbReader(pdbFile)) { + try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) { final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError()); Assert.assertEquals(actualA, entryA); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java index 6ce7902..2a03739 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -111,7 +111,8 @@ public class PerformanceDbTest { final List actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList(); Assert.assertEquals(actualEntries, entries); - final List filesInStorage = FileUtils.listRecursively(DataStore.storageDirectory(dataDirectory)); + final Path storageBasePath = DataStore.storageDirectory(dataDirectory); + final List filesInStorage = FileUtils.listRecursively(storageBasePath); Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv"); @@ -119,7 +120,7 @@ public class PerformanceDbTest { final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags); - try (PdbReader pdbReader = new PdbReader(pdbFile)) { + try (PdbReader pdbReader = new PdbReader(storageBasePath, pdbFile)) { Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0)); Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1)); Assert.assertEquals(pdbReader.readEntry().isPresent(), false); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java index 8eeabf1..0639b2b 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -31,8 +31,8 @@ public class TagsToFilesTest { public void test() throws Exception { - final PdbDB db = new PdbDB(dataDirectory); - try (final TagsToFile tagsToFile = new TagsToFile(db)) { + try (final PdbDB db = new PdbDB(dataDirectory); // + final TagsToFile tagsToFile = new TagsToFile(db)) { final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); final Tags tags = Tags.create("myKey", "myValue"); @@ -46,8 +46,9 @@ public class TagsToFilesTest { } public void testAppendingToSameFileIfNewDateIsAfter() throws Exception { - final PdbDB db = new PdbDB(dataDirectory); - try (final TagsToFile tagsToFile = new TagsToFile(db);) { + + try (final PdbDB db = new PdbDB(dataDirectory); // + final TagsToFile tagsToFile = new TagsToFile(db);) { final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1); final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1); @@ -66,8 +67,8 @@ public class TagsToFilesTest { @Test(invocationCount = 1) public void testNewFileIfDateIsTooOld() throws Exception { - final PdbDB db = new PdbDB(dataDirectory); - try (final TagsToFile tagsToFile = new TagsToFile(db);) { + try (final PdbDB db = new PdbDB(dataDirectory); // + final TagsToFile tagsToFile = new TagsToFile(db);) { final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1); @@ -104,8 +105,8 @@ public class TagsToFilesTest { public void testIdenticalDatesGoIntoSameFile() throws Exception { - final PdbDB db = new PdbDB(dataDirectory); - try (final TagsToFile tagsToFile = new TagsToFile(db)) { + try (final PdbDB db = new PdbDB(dataDirectory); // + final TagsToFile tagsToFile = new TagsToFile(db)) { final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);