only store the tag specific base folder in the database

before that we added each file (one per day and tag combination) to the
db

performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java

@@ -6,8 +6,12 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+import java.util.function.BiPredicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class FileUtils {
 	private static final Logger LOGGER = Logger.getLogger(FileUtils.class.getCanonicalName());
@@ -52,4 +56,16 @@ public class FileUtils {
 			attempt++;
 		}
 	}
+
+	public static List<Path> listRecursively(final Path start) {
+
+		final int maxDepth = Integer.MAX_VALUE;
+		final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
+
+		try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
+			return files.collect(Collectors.toList());
+		} catch (final IOException e) {
+			throw new ReadException(e);
+		}
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java

@@ -1,6 +1,7 @@
 package org.lucares.performance.db;
 
 import java.io.File;
+import java.nio.file.Path;
 
 import org.lucares.pdb.api.Tags;
 
@@ -9,13 +10,21 @@ class PdbFile {
 
 	private final Day day;
 
-	private final File file;
+	private final Path path;
 
 	private final long offsetInEpochMilli;
 
+	@Deprecated
 	public PdbFile(final Day day, final File file, final Tags tags) {
 		this.day = day;
-		this.file = file;
+		this.path = file.toPath();
+		this.tags = tags;
+		offsetInEpochMilli = day.getOffsetInEpochMilli();
+	}
+
+	public PdbFile(final Day day, final Path path, final Tags tags) {
+		this.day = day;
+		this.path = path;
 		this.tags = tags;
 		offsetInEpochMilli = day.getOffsetInEpochMilli();
 	}
@@ -29,8 +38,13 @@ class PdbFile {
 		return tags;
 	}
 
+	@Deprecated
 	public File getFile() {
-		return file;
+		return path.toFile();
+	}
+
+	public Path getPath() {
+		return path;
 	}
 
 	public Day getDay() {
@@ -48,7 +62,7 @@ class PdbFile {
 
 	@Override
 	public String toString() {
-		return "PdbFile [" + file + " " + getTimeRange() + " " + tags + "]\n";
+		return "PdbFile [" + path + " " + getTimeRange() + " " + tags + "]\n";
 	}
 
 	@Override
@@ -56,7 +70,8 @@ class PdbFile {
 		final int prime = 31;
 		int result = 1;
 		result = prime * result + ((day == null) ? 0 : day.hashCode());
-		result = prime * result + ((file == null) ? 0 : file.hashCode());
+		result = prime * result + (int) (offsetInEpochMilli ^ (offsetInEpochMilli >>> 32));
+		result = prime * result + ((path == null) ? 0 : path.hashCode());
 		result = prime * result + ((tags == null) ? 0 : tags.hashCode());
 		return result;
 	}
@@ -75,10 +90,12 @@ class PdbFile {
 				return false;
 		} else if (!day.equals(other.day))
 			return false;
-		if (file == null) {
+		if (offsetInEpochMilli != other.offsetInEpochMilli)
-			if (other.file != null)
 			return false;
-		} else if (!file.equals(other.file))
+		if (path == null) {
+			if (other.path != null)
+				return false;
+		} else if (!path.equals(other.path))
 			return false;
 		if (tags == null) {
 			if (other.tags != null)
@@ -87,5 +104,4 @@ class PdbFile {
 			return false;
 		return true;
 	}
-
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java

@@ -1,6 +1,8 @@
 package org.lucares.performance.db;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
 import java.util.ArrayDeque;
 import java.util.Collection;
 import java.util.Iterator;
@@ -63,14 +65,17 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
 			while (!pdbFiles.isEmpty()) {
 				currentPdbFile = pdbFiles.poll();
 				try {
-					if (currentPdbFile.getFile().length() > 0) {
+
+					if (Files.size(currentPdbFile.getPath()) > 0) {
 						reader = new PdbReader(currentPdbFile);
 						break;
 					} else {
 						LOGGER.info("ignoring empty file " + currentPdbFile);
 					}
 				} catch (final FileNotFoundException e) {
-					LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e);
+					LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e);
+				} catch (final IOException e) {
+					throw new ReadException(e);
 				}
 			}
 		}

performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java

@@ -1,5 +1,6 @@
 package org.lucares.performance.db;
 
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
@@ -21,7 +22,8 @@ class PdbReader implements AutoCloseable {
 	public PdbReader(final PdbFile pdbFile) throws FileNotFoundException {
 		super();
 		this.pdbFile = pdbFile;
-		this.data = new RandomAccessFile(pdbFile.getFile(), "r");
+		File storageFile = pdbFile.getPath().toFile();
+		this.data = new RandomAccessFile(storageFile, "r");
 	}
 
 	/**

performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java

@@ -1,6 +1,7 @@
 package org.lucares.performance.db;
 
 import java.io.BufferedOutputStream;
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -19,9 +20,10 @@ class PdbWriter implements AutoCloseable {
 
 	PdbWriter(final PdbFile pdbFile) throws IOException {
 		this.pdbFile = pdbFile;
-		this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND));
+		final File storageFile = pdbFile.getPath().toFile();
+		this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
 
-		if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) {
+		if (storageFile.exists() && storageFile.length() > 0) {
 			final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
 			minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli();
 		} else {

performanceDb/src/main/java/org/lucares/performance/db/ReadException.java

@@ -0,0 +1,12 @@
+package org.lucares.performance.db;
+
+import java.io.IOException;
+
+public class ReadException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+
+	public ReadException(final IOException e) {
+		super(e);
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -1,13 +1,11 @@
 package org.lucares.performance.db;
 
-import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -18,6 +16,26 @@ import org.lucares.pdb.api.Tags;
 
 public class TagsToFile implements CollectionUtils {
 
+	private static class TagSpecificBaseDir {
+		private final Path path;
+
+		private final Tags tags;
+
+		public TagSpecificBaseDir(final Path path, final Tags tags) {
+			super();
+			this.path = path;
+			this.tags = tags;
+		}
+
+		public Path getPath() {
+			return path;
+		}
+
+		public Tags getTags() {
+			return tags;
+		}
+	}
+
 	private final H2DB db;
 	private final Path dataDirectory;
 
@@ -32,38 +50,52 @@ public class TagsToFile implements CollectionUtils {
 		return filter(files, f -> f.getTags().equals(tags));
 	}
 
-	List<PdbFile> getFilesMatchingTags(final Tags tags) {
+	private List<PdbFile> getFilesMatchingTags(final Tags tags) {
 		final String query = Query.createQuery(tags);
+
 		return getFilesForQuery(query);
 	}
 
 	List<PdbFile> getFilesForQuery(final String query) {
 		final List<PdbFile> result = new ArrayList<>();
+		final List<TagSpecificBaseDir> tagSpecificFolders = getTagSpecificFolders(query);
+
+		for (final TagSpecificBaseDir tagSpecific : tagSpecificFolders) {
+
+			final Tags fileSpecificTags = tagSpecific.getTags();
+
+			final List<Path> storageFiles = FileUtils.listRecursively(tagSpecific.getPath());
+			for (final Path storageFile : storageFiles) {
+
+				final Day day = StorageUtils.getDateOffset(storageFile);
+				final PdbFile pdbFile = new PdbFile(day, storageFile, fileSpecificTags);
+
+				result.add(pdbFile);
+			}
+		}
+
+		return result;
+	}
+
+	private List<TagSpecificBaseDir> getTagSpecificFolders(final String query) {
+		final List<TagSpecificBaseDir> result = new ArrayList<>();
 		try {
 			final List<Document> searchResult = db.search(query);
 
 			for (final Document document : searchResult) {
 
-				final PdbFile pdbFile = toPdbFile(document);
+				final Path path = document.getFile().toPath();
+				final Tags tags = toTags(document);
 
-				result.add(pdbFile);
+				result.add(new TagSpecificBaseDir(path, tags));
 			}
 		} catch (final NullPointerException e) {
 			// TODO @ahr unknown fields in searches must be handled better
 		}
 
-		Collections.sort(result, PdbFileByTimeAsc.INSTANCE);
 		return result;
 	}
 
-	private PdbFile toPdbFile(final Document document) {
-		final File file = document.getFile();
-		final Day day = StorageUtils.getDateOffset(file.toPath());
-		final Tags tagsOfFile = toTags(document);
-		final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile);
-		return pdbFile;
-	}
-
 	private Tags toTags(final Document document) {
 		Tags tagsOfFile = Tags.create();
 
@@ -103,7 +135,7 @@ public class TagsToFile implements CollectionUtils {
 			if (pdbFiles.isEmpty()) {
 				tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
 			} else {
-				final Path storageFilePath = pdbFiles.get(0).getFile().toPath();
+				final Path storageFilePath = pdbFiles.get(0).getPath();
 				tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
 			}
 
@@ -116,9 +148,9 @@ public class TagsToFile implements CollectionUtils {
 	}
 
 	private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
-		final Set<File> reducedFolder = pdbFiles.stream()//
+		final Set<Path> reducedFolder = pdbFiles.stream()//
-				.map(PdbFile::getFile)//
+				.map(PdbFile::getPath)//
-				.map(f -> f.getParentFile().getParentFile().getParentFile())//
+				.map(p -> p.getParent().getParent().getParent())//
 				.collect(Collectors.toSet());
 
 		if (reducedFolder.size() > 1) {
@@ -128,22 +160,28 @@ public class TagsToFile implements CollectionUtils {
 	}
 
 	private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) {
-		final File file;
+		final Path storageFile;
 		PdbFile result;
-		file = createNewFile(date, tagSpecificStorageFolder);
+		storageFile = createNewFile(date, tagSpecificStorageFolder);
 		final Day day = new Day(date);
 
-		db.addDocument(file);
+		final Document document = db.getDocument(tagSpecificStorageFolder.toFile());
+		if (document == null) {
+			System.err.println("adding new folder: " + tagSpecificStorageFolder);
+			db.addDocument(tagSpecificStorageFolder.toFile());
 
 			tags.forEach((fieldName, value) -> {
-			TagsUtils.setProperty(db, file, fieldName, value);
+				TagsUtils.setProperty(db, tagSpecificStorageFolder.toFile(), fieldName, value);
 			});
+		} else {
+			System.out.println("using existing folder: " + tagSpecificStorageFolder);
+		}
 
-		result = new PdbFile(day, file, tags);
+		result = new PdbFile(day, storageFile, tags);
 		return result;
 	}
 
-	private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
+	private Path createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
 		final Day day = new Day(date);
 
 		final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
@@ -154,6 +192,6 @@ public class TagsToFile implements CollectionUtils {
 			throw new IllegalStateException(e); // very unlikely
 		}
 
-		return result.toFile();
+		return result;
 	}
 }

performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java

@@ -1,6 +1,5 @@
 package org.lucares.performance.db;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -32,10 +31,10 @@ public class PdbWriterManagerTest {
 	public void testManager() throws Exception {
 
 		final PdbWriterSupplier supplier = (tags, date) -> {
-			File file;
+			Path path;
 			try {
-				file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile();
+				path = Files.createTempFile(dataDirectory, "pdb", ".data");
-				return new PdbWriter(new PdbFile(new Day(date), file, tags));
+				return new PdbWriter(new PdbFile(new Day(date), path, tags));
 			} catch (final IOException e) {
 				throw new AssertionError(e.getMessage(), e);
 			}