From 580733d267bd2ced4c240afe76400e29e98ec954 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 23 Dec 2016 19:12:30 +0100 Subject: [PATCH] only store the tag specific base folder in the database before that we added each file (one per day and tag combination) to the db --- .../org/lucares/performance/db/FileUtils.java | 16 ++++ .../org/lucares/performance/db/PdbFile.java | 34 +++++-- .../performance/db/PdbFileIterator.java | 9 +- .../org/lucares/performance/db/PdbReader.java | 4 +- .../org/lucares/performance/db/PdbWriter.java | 6 +- .../lucares/performance/db/ReadException.java | 12 +++ .../lucares/performance/db/TagsToFile.java | 92 +++++++++++++------ .../performance/db/PdbWriterManagerTest.java | 7 +- 8 files changed, 135 insertions(+), 45 deletions(-) create mode 100644 performanceDb/src/main/java/org/lucares/performance/db/ReadException.java diff --git a/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java index e9540ab..f05ec99 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java @@ -6,8 +6,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; +import java.util.function.BiPredicate; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class FileUtils { private static final Logger LOGGER = Logger.getLogger(FileUtils.class.getCanonicalName()); @@ -52,4 +56,16 @@ public class FileUtils { attempt++; } } + + public static List listRecursively(final Path start) { + + final int maxDepth = Integer.MAX_VALUE; + final BiPredicate matcher = (path, attr) -> Files.isRegularFile(path); + + try (final Stream files = Files.find(start, maxDepth, matcher)) { + return files.collect(Collectors.toList()); + } catch (final IOException e) { + throw new ReadException(e); + } + } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java index 54f93f9..65a0dcc 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java @@ -1,6 +1,7 @@ package org.lucares.performance.db; import java.io.File; +import java.nio.file.Path; import org.lucares.pdb.api.Tags; @@ -9,13 +10,21 @@ class PdbFile { private final Day day; - private final File file; + private final Path path; private final long offsetInEpochMilli; + @Deprecated public PdbFile(final Day day, final File file, final Tags tags) { this.day = day; - this.file = file; + this.path = file.toPath(); + this.tags = tags; + offsetInEpochMilli = day.getOffsetInEpochMilli(); + } + + public PdbFile(final Day day, final Path path, final Tags tags) { + this.day = day; + this.path = path; this.tags = tags; offsetInEpochMilli = day.getOffsetInEpochMilli(); } @@ -29,8 +38,13 @@ class PdbFile { return tags; } + @Deprecated public File getFile() { - return file; + return path.toFile(); + } + + public Path getPath() { + return path; } public Day getDay() { @@ -48,7 +62,7 @@ class PdbFile { @Override public String toString() { - return "PdbFile [" + file + " " + getTimeRange() + " " + tags + "]\n"; + return "PdbFile [" + path + " " + getTimeRange() + " " + tags + "]\n"; } @Override @@ -56,7 +70,8 @@ class PdbFile { final int prime = 31; int result = 1; result = prime * result + ((day == null) ? 0 : day.hashCode()); - result = prime * result + ((file == null) ? 0 : file.hashCode()); + result = prime * result + (int) (offsetInEpochMilli ^ (offsetInEpochMilli >>> 32)); + result = prime * result + ((path == null) ? 0 : path.hashCode()); result = prime * result + ((tags == null) ? 0 : tags.hashCode()); return result; } @@ -75,10 +90,12 @@ class PdbFile { return false; } else if (!day.equals(other.day)) return false; - if (file == null) { - if (other.file != null) + if (offsetInEpochMilli != other.offsetInEpochMilli) + return false; + if (path == null) { + if (other.path != null) return false; - } else if (!file.equals(other.file)) + } else if (!path.equals(other.path)) return false; if (tags == null) { if (other.tags != null) @@ -87,5 +104,4 @@ class PdbFile { return false; return true; } - } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index 5477e4a..ce16921 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -1,6 +1,8 @@ package org.lucares.performance.db; import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayDeque; import java.util.Collection; import java.util.Iterator; @@ -63,14 +65,17 @@ public class PdbFileIterator implements Iterator, AutoCloseable { while (!pdbFiles.isEmpty()) { currentPdbFile = pdbFiles.poll(); try { - if (currentPdbFile.getFile().length() > 0) { + + if (Files.size(currentPdbFile.getPath()) > 0) { reader = new PdbReader(currentPdbFile); break; } else { LOGGER.info("ignoring empty file " + currentPdbFile); } } catch (final FileNotFoundException e) { - LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e); + LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e); + } catch (final IOException e) { + throw new ReadException(e); } } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index 9f622ea..b4a37d1 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -1,5 +1,6 @@ package org.lucares.performance.db; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; @@ -21,7 +22,8 @@ class PdbReader implements AutoCloseable { public PdbReader(final PdbFile pdbFile) throws FileNotFoundException { super(); this.pdbFile = pdbFile; - this.data = new RandomAccessFile(pdbFile.getFile(), "r"); + File storageFile = pdbFile.getPath().toFile(); + this.data = new RandomAccessFile(storageFile, "r"); } /** diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java index 816f3ae..6c3bdd2 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -1,6 +1,7 @@ package org.lucares.performance.db; import java.io.BufferedOutputStream; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -19,9 +20,10 @@ class PdbWriter implements AutoCloseable { PdbWriter(final PdbFile pdbFile) throws IOException { this.pdbFile = pdbFile; - this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND)); + final File storageFile = pdbFile.getPath().toFile(); + this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND)); - if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) { + if (storageFile.exists() && storageFile.length() > 0) { final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile); minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli(); } else { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/ReadException.java b/performanceDb/src/main/java/org/lucares/performance/db/ReadException.java new file mode 100644 index 0000000..dd4447d --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/ReadException.java @@ -0,0 +1,12 @@ +package org.lucares.performance.db; + +import java.io.IOException; + +public class ReadException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public ReadException(final IOException e) { + super(e); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index 98a7003..ddc60c5 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -1,13 +1,11 @@ package org.lucares.performance.db; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -18,6 +16,26 @@ import org.lucares.pdb.api.Tags; public class TagsToFile implements CollectionUtils { + private static class TagSpecificBaseDir { + private final Path path; + + private final Tags tags; + + public TagSpecificBaseDir(final Path path, final Tags tags) { + super(); + this.path = path; + this.tags = tags; + } + + public Path getPath() { + return path; + } + + public Tags getTags() { + return tags; + } + } + private final H2DB db; private final Path dataDirectory; @@ -32,38 +50,52 @@ public class TagsToFile implements CollectionUtils { return filter(files, f -> f.getTags().equals(tags)); } - List getFilesMatchingTags(final Tags tags) { + private List getFilesMatchingTags(final Tags tags) { final String query = Query.createQuery(tags); + return getFilesForQuery(query); } List getFilesForQuery(final String query) { final List result = new ArrayList<>(); + final List tagSpecificFolders = getTagSpecificFolders(query); + + for (final TagSpecificBaseDir tagSpecific : tagSpecificFolders) { + + final Tags fileSpecificTags = tagSpecific.getTags(); + + final List storageFiles = FileUtils.listRecursively(tagSpecific.getPath()); + for (final Path storageFile : storageFiles) { + + final Day day = StorageUtils.getDateOffset(storageFile); + final PdbFile pdbFile = new PdbFile(day, storageFile, fileSpecificTags); + + result.add(pdbFile); + } + } + + return result; + } + + private List getTagSpecificFolders(final String query) { + final List result = new ArrayList<>(); try { final List searchResult = db.search(query); for (final Document document : searchResult) { - final PdbFile pdbFile = toPdbFile(document); + final Path path = document.getFile().toPath(); + final Tags tags = toTags(document); - result.add(pdbFile); + result.add(new TagSpecificBaseDir(path, tags)); } } catch (final NullPointerException e) { // TODO @ahr unknown fields in searches must be handled better } - Collections.sort(result, PdbFileByTimeAsc.INSTANCE); return result; } - private PdbFile toPdbFile(final Document document) { - final File file = document.getFile(); - final Day day = StorageUtils.getDateOffset(file.toPath()); - final Tags tagsOfFile = toTags(document); - final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile); - return pdbFile; - } - private Tags toTags(final Document document) { Tags tagsOfFile = Tags.create(); @@ -103,7 +135,7 @@ public class TagsToFile implements CollectionUtils { if (pdbFiles.isEmpty()) { tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags); } else { - final Path storageFilePath = pdbFiles.get(0).getFile().toPath(); + final Path storageFilePath = pdbFiles.get(0).getPath(); tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath); } @@ -116,9 +148,9 @@ public class TagsToFile implements CollectionUtils { } private void assertAllFilesHaveSameFolder(final List pdbFiles) { - final Set reducedFolder = pdbFiles.stream()// - .map(PdbFile::getFile)// - .map(f -> f.getParentFile().getParentFile().getParentFile())// + final Set reducedFolder = pdbFiles.stream()// + .map(PdbFile::getPath)// + .map(p -> p.getParent().getParent().getParent())// .collect(Collectors.toSet()); if (reducedFolder.size() > 1) { @@ -128,22 +160,28 @@ public class TagsToFile implements CollectionUtils { } private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) { - final File file; + final Path storageFile; PdbFile result; - file = createNewFile(date, tagSpecificStorageFolder); + storageFile = createNewFile(date, tagSpecificStorageFolder); final Day day = new Day(date); - db.addDocument(file); + final Document document = db.getDocument(tagSpecificStorageFolder.toFile()); + if (document == null) { + System.err.println("adding new folder: " + tagSpecificStorageFolder); + db.addDocument(tagSpecificStorageFolder.toFile()); - tags.forEach((fieldName, value) -> { - TagsUtils.setProperty(db, file, fieldName, value); - }); + tags.forEach((fieldName, value) -> { + TagsUtils.setProperty(db, tagSpecificStorageFolder.toFile(), fieldName, value); + }); + } else { + System.out.println("using existing folder: " + tagSpecificStorageFolder); + } - result = new PdbFile(day, file, tags); + result = new PdbFile(day, storageFile, tags); return result; } - private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) { + private Path createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) { final Day day = new Day(date); final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day); @@ -154,6 +192,6 @@ public class TagsToFile implements CollectionUtils { throw new IllegalStateException(e); // very unlikely } - return result.toFile(); + return result; } } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java index e86e505..b5e8851 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java @@ -1,6 +1,5 @@ package org.lucares.performance.db; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -32,10 +31,10 @@ public class PdbWriterManagerTest { public void testManager() throws Exception { final PdbWriterSupplier supplier = (tags, date) -> { - File file; + Path path; try { - file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile(); - return new PdbWriter(new PdbFile(new Day(date), file, tags)); + path = Files.createTempFile(dataDirectory, "pdb", ".data"); + return new PdbWriter(new PdbFile(new Day(date), path, tags)); } catch (final IOException e) { throw new AssertionError(e.getMessage(), e); }