andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

only store the tag specific base folder in the database

Browse Source 
before that we added each file (one per day and tag combination) to the
db
This commit is contained in:
Andreas Huber
2016-12-23 19:12:30 +01:00
parent 6969c8ce46
commit 580733d267
8 changed files with 135 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java
View File

@@ -6,8 +6,12 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.function.BiPredicate;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class FileUtils {
private static final Logger LOGGER = Logger.getLogger(FileUtils.class.getCanonicalName());
@@ -52,4 +56,16 @@ public class FileUtils {
attempt++;
}
}
public static List<Path> listRecursively(final Path start) {
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
return files.collect(Collectors.toList());
} catch (final IOException e) {
throw new ReadException(e);
}
}
}

34
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -1,6 +1,7 @@
package org.lucares.performance.db;
import java.io.File;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
@@ -9,13 +10,21 @@ class PdbFile {
private final Day day;
private final File file;
private final Path path;
private final long offsetInEpochMilli;
@Deprecated
public PdbFile(final Day day, final File file, final Tags tags) {
this.day = day;
this.file = file;
this.path = file.toPath();
this.tags = tags;
offsetInEpochMilli = day.getOffsetInEpochMilli();
}
public PdbFile(final Day day, final Path path, final Tags tags) {
this.day = day;
this.path = path;
this.tags = tags;
offsetInEpochMilli = day.getOffsetInEpochMilli();
}
@@ -29,8 +38,13 @@ class PdbFile {
return tags;
}
@Deprecated
public File getFile() {
return file;
return path.toFile();
}
public Path getPath() {
return path;
}
public Day getDay() {
@@ -48,7 +62,7 @@ class PdbFile {
@Override
public String toString() {
return "PdbFile [" + file + " " + getTimeRange() + " " + tags + "]\n";
return "PdbFile [" + path + " " + getTimeRange() + " " + tags + "]\n";
}
@Override
@@ -56,7 +70,8 @@ class PdbFile {
final int prime = 31;
int result = 1;
result = prime * result + ((day == null) ? 0 : day.hashCode());
result = prime * result + ((file == null) ? 0 : file.hashCode());
result = prime * result + (int) (offsetInEpochMilli ^ (offsetInEpochMilli >>> 32));
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@@ -75,10 +90,12 @@ class PdbFile {
return false;
} else if (!day.equals(other.day))
return false;
if (file == null) {
if (other.file != null)
if (offsetInEpochMilli != other.offsetInEpochMilli)
return false;
if (path == null) {
if (other.path != null)
return false;
} else if (!file.equals(other.file))
} else if (!path.equals(other.path))
return false;
if (tags == null) {
if (other.tags != null)
@@ -87,5 +104,4 @@ class PdbFile {
return false;
return true;
}
}

9
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -1,6 +1,8 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
@@ -63,14 +65,17 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
while (!pdbFiles.isEmpty()) {
currentPdbFile = pdbFiles.poll();
try {
if (currentPdbFile.getFile().length() > 0) {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e);
LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
}
}

4
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -1,5 +1,6 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
@@ -21,7 +22,8 @@ class PdbReader implements AutoCloseable {
public PdbReader(final PdbFile pdbFile) throws FileNotFoundException {
super();
this.pdbFile = pdbFile;
this.data = new RandomAccessFile(pdbFile.getFile(), "r");
File storageFile = pdbFile.getPath().toFile();
this.data = new RandomAccessFile(storageFile, "r");
}
/**

6
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -1,6 +1,7 @@
package org.lucares.performance.db;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -19,9 +20,10 @@ class PdbWriter implements AutoCloseable {
PdbWriter(final PdbFile pdbFile) throws IOException {
this.pdbFile = pdbFile;
this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND));
final File storageFile = pdbFile.getPath().toFile();
this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) {
if (storageFile.exists() && storageFile.length() > 0) {
final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli();
} else {

12
performanceDb/src/main/java/org/lucares/performance/db/ReadException.java Normal file
View File

@@ -0,0 +1,12 @@
package org.lucares.performance.db;
import java.io.IOException;
public class ReadException extends RuntimeException {
private static final long serialVersionUID = 1L;
public ReadException(final IOException e) {
super(e);
}
}

92
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,13 +1,11 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@@ -18,6 +16,26 @@ import org.lucares.pdb.api.Tags;
public class TagsToFile implements CollectionUtils {
private static class TagSpecificBaseDir {
private final Path path;
private final Tags tags;
public TagSpecificBaseDir(final Path path, final Tags tags) {
super();
this.path = path;
this.tags = tags;
}
public Path getPath() {
return path;
}
public Tags getTags() {
return tags;
}
}
private final H2DB db;
private final Path dataDirectory;
@@ -32,38 +50,52 @@ public class TagsToFile implements CollectionUtils {
return filter(files, f -> f.getTags().equals(tags));
}
List<PdbFile> getFilesMatchingTags(final Tags tags) {
private List<PdbFile> getFilesMatchingTags(final Tags tags) {
final String query = Query.createQuery(tags);
return getFilesForQuery(query);
}
List<PdbFile> getFilesForQuery(final String query) {
final List<PdbFile> result = new ArrayList<>();
final List<TagSpecificBaseDir> tagSpecificFolders = getTagSpecificFolders(query);
for (final TagSpecificBaseDir tagSpecific : tagSpecificFolders) {
final Tags fileSpecificTags = tagSpecific.getTags();
final List<Path> storageFiles = FileUtils.listRecursively(tagSpecific.getPath());
for (final Path storageFile : storageFiles) {
final Day day = StorageUtils.getDateOffset(storageFile);
final PdbFile pdbFile = new PdbFile(day, storageFile, fileSpecificTags);
result.add(pdbFile);
}
}
return result;
}
private List<TagSpecificBaseDir> getTagSpecificFolders(final String query) {
final List<TagSpecificBaseDir> result = new ArrayList<>();
try {
final List<Document> searchResult = db.search(query);
for (final Document document : searchResult) {
final PdbFile pdbFile = toPdbFile(document);
final Path path = document.getFile().toPath();
final Tags tags = toTags(document);
result.add(pdbFile);
result.add(new TagSpecificBaseDir(path, tags));
}
} catch (final NullPointerException e) {
// TODO @ahr unknown fields in searches must be handled better
}
Collections.sort(result, PdbFileByTimeAsc.INSTANCE);
return result;
}
private PdbFile toPdbFile(final Document document) {
final File file = document.getFile();
final Day day = StorageUtils.getDateOffset(file.toPath());
final Tags tagsOfFile = toTags(document);
final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile);
return pdbFile;
}
private Tags toTags(final Document document) {
Tags tagsOfFile = Tags.create();
@@ -103,7 +135,7 @@ public class TagsToFile implements CollectionUtils {
if (pdbFiles.isEmpty()) {
tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
} else {
final Path storageFilePath = pdbFiles.get(0).getFile().toPath();
final Path storageFilePath = pdbFiles.get(0).getPath();
tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
}
@@ -116,9 +148,9 @@ public class TagsToFile implements CollectionUtils {
}
private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
final Set<File> reducedFolder = pdbFiles.stream()//
.map(PdbFile::getFile)//
.map(f -> f.getParentFile().getParentFile().getParentFile())//
final Set<Path> reducedFolder = pdbFiles.stream()//
.map(PdbFile::getPath)//
.map(p -> p.getParent().getParent().getParent())//
.collect(Collectors.toSet());
if (reducedFolder.size() > 1) {
@@ -128,22 +160,28 @@ public class TagsToFile implements CollectionUtils {
}
private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) {
final File file;
final Path storageFile;
PdbFile result;
file = createNewFile(date, tagSpecificStorageFolder);
storageFile = createNewFile(date, tagSpecificStorageFolder);
final Day day = new Day(date);
db.addDocument(file);
final Document document = db.getDocument(tagSpecificStorageFolder.toFile());
if (document == null) {
System.err.println("adding new folder: " + tagSpecificStorageFolder);
db.addDocument(tagSpecificStorageFolder.toFile());
tags.forEach((fieldName, value) -> {
TagsUtils.setProperty(db, file, fieldName, value);
});
tags.forEach((fieldName, value) -> {
TagsUtils.setProperty(db, tagSpecificStorageFolder.toFile(), fieldName, value);
});
} else {
System.out.println("using existing folder: " + tagSpecificStorageFolder);
}
result = new PdbFile(day, file, tags);
result = new PdbFile(day, storageFile, tags);
return result;
}
private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
private Path createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
final Day day = new Day(date);
final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
@@ -154,6 +192,6 @@ public class TagsToFile implements CollectionUtils {
throw new IllegalStateException(e); // very unlikely
}
return result.toFile();
return result;
}
}

7
performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java
View File

@@ -1,6 +1,5 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -32,10 +31,10 @@ public class PdbWriterManagerTest {
public void testManager() throws Exception {
final PdbWriterSupplier supplier = (tags, date) -> {
File file;
Path path;
try {
file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile();
return new PdbWriter(new PdbFile(new Day(date), file, tags));
path = Files.createTempFile(dataDirectory, "pdb", ".data");
return new PdbWriter(new PdbFile(new Day(date), path, tags));
} catch (final IOException e) {
throw new AssertionError(e.getMessage(), e);
}