andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

replace ludb with data-store

Browse Source 
LuDB has a few disadvantages. 
  1. Most notably disk space. H2 wastes a lot of valuable disk space.
     For my test data set with 44 million entries it is 14 MB 
     (sometimes a lot more; depends on H2 internal cleanup). With 
     data-store it is 15 KB.
     Overall I could reduce the disk space from 231 MB to 200 MB (13.4 %
     in this example). That is an average of 4.6 bytes per entry.
  2. Speed:
     a) Liquibase is slow. The first time it takes approx. three seconds
     b) Query and insertion. with data-store we can insert entries 
        up to 1.6 times faster.

Data-store uses a few tricks to save disk space:
  1. We encode the tags into the file names.
  2. To keep them short we translate the key/value of the tag into 
     shorter numbers. For example "foo" -> 12 and "bar" to 47. So the
     tag "foo"/"bar" would be 12/47. 
     We then translate this number into a numeral system of base 62
     (a-zA-Z0-9), so it can be used for file names and it is shorter.
     That way we only have to store the mapping of string to int.
  3. We do that in a simple tab separated file.
This commit is contained in:
Andreas Huber
2017-04-16 09:07:28 +02:00
parent 85e45f74b7
commit ac1ee20046
56 changed files with 2243 additions and 677 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
performanceDb/build.gradle
View File

@@ -1,9 +1,11 @@
dependencies {
compile project(':pdb-api')
compile 'org.lucares:ludb:1.0.20170408081113'
compile project(':data-store')
compile project(':file-utils')
//compile 'org.lucares:ludb:1.0.20170408081113'
compile 'com.fasterxml.jackson.core:jackson-databind:2.8.8'
compile 'org.apache.commons:commons-collections4:4.1'
compile 'org.apache.logging.log4j:log4j-api:2.8.2'

42
performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java
View File

@@ -1,42 +0,0 @@
package org.lucares.performance.db;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public interface CollectionUtils {
public default <T, V> Map<T, V> toMap(final Iterable<V> iterable, final Function<V, T> keyMapper) {
final Map<T, V> result = new HashMap<>();
for (final V value : iterable) {
final T key = keyMapper.apply(value);
result.put(key, value);
}
return result;
}
public default <T> List<T> filter(final Collection<T> collection, final Predicate<T> predicate) {
return collection.stream().filter(predicate).collect(Collectors.toList());
}
public default <T, R> List<R> map(final Collection<T> collection, final Function<T, R> mapper) {
return collection.stream().map(mapper).collect(Collectors.toList());
}
public default <T> List<T> sorted(final Collection<T> collection, final Comparator<T> comparator) {
return collection.stream().sorted(comparator).collect(Collectors.toList());
}
public default <T> Optional<T> findFirst(final Collection<T> collection) {
return collection.stream().findFirst();
}
}

72
performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java
View File

@@ -1,72 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.function.BiPredicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FileUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
private static final class RecursiveDeleter extends SimpleFileVisitor<Path> {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
Files.delete(file);
LOGGER.trace("deleted: {}", file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
Files.delete(dir);
LOGGER.trace("deleted: {}", dir);
return FileVisitResult.CONTINUE;
}
}
public static void delete(final Path path) {
final int maxAttempts = 10;
int attempt = 1;
while (attempt <= maxAttempts) {
try {
LOGGER.debug("deleting '{}' attempt {} of {}", path.toFile().getAbsolutePath(), attempt, maxAttempts);
Files.walkFileTree(path, new RecursiveDeleter());
break;
} catch (final IOException e) {
final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt
+ " of " + maxAttempts;
LOGGER.warn(msg, e);
}
attempt++;
}
}
public static List<Path> listRecursively(final Path start) {
final int maxDepth = Integer.MAX_VALUE;
final BiPredicate<Path, BasicFileAttributes> matcher = (path, attr) -> Files.isRegularFile(path);
try (final Stream<Path> files = Files.find(start, maxDepth, matcher)) {
return files.collect(Collectors.toList());
} catch (final IOException e) {
throw new ReadException(e);
}
}
}

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -8,7 +8,6 @@ import java.util.Queue;
import java.util.function.Supplier;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
@@ -31,15 +30,14 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader == null) {
return null;
}
final Entry entry = reader.readNullableEntry(reader.getPdbFile().getTags());
final Entry entry = reader.readNullableEntry();
if (entry == null) {
nextFile();
if (reader == null) {
return null;
} else {
final Tags tags = reader.getPdbFile().getTags();
return reader.readEntry(tags).orElse(null);
return reader.readEntry().orElse(null);
}
}

29
performanceDb/src/main/java/org/lucares/performance/db/PdbFileOffsetTime.java
View File

@@ -1,29 +0,0 @@
package org.lucares.performance.db;
import java.time.OffsetDateTime;
public class PdbFileOffsetTime {
private final PdbFile pdbFile;
private final OffsetDateTime offsetTime;
public PdbFileOffsetTime(final PdbFile pdbFile, final OffsetDateTime offsetTime) {
super();
this.pdbFile = pdbFile;
this.offsetTime = offsetTime;
}
public PdbFile getPdbFile() {
return pdbFile;
}
public OffsetDateTime getOffsetTime() {
return offsetTime;
}
@Override
public String toString() {
return "PdbFileOffsetTime [pdbFile=" + pdbFile + ", offsetTime=" + offsetTime + "]";
}
}

30
performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java
View File

@@ -13,6 +13,7 @@ public class PdbFileViewer {
final File file = new File(args[0]);
final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
long countMeasurements = 0;
try (final PdbReader reader = new PdbReader(pdbFile, false)) {
long value = 0;
@@ -20,6 +21,7 @@ public class PdbFileViewer {
while ((nextByte = reader.readNextByte()) >= 0) {
final ByteType type = ByteType.getType(nextByte);
countMeasurements = countMeasurements + (type == ByteType.MEASUREMENT ? 1 : 0);
final long bytesValue = type.getValue(nextByte);
if (type == ByteType.CONTINUATION) {
@@ -29,10 +31,36 @@ public class PdbFileViewer {
value = bytesValue;
}
System.out.printf("%s %3d %3d %-14s %14d\n", toBinary(nextByte), nextByte, bytesValue, type, value);
String additionalInfo = "";
if (ByteType.MEASUREMENT == ByteType.getType(reader.peekNextByte())) {
additionalInfo = format(value);
}
System.out.printf("%s %3d %3d %-14s %14d %s\n", toBinary(nextByte), nextByte, bytesValue, type, value,
additionalInfo);
}
}
System.out.println("Bytes: " + file.length());
System.out.println("Measurements: " + countMeasurements);
System.out.println("Bytes/Measurements: " + (file.length() / (double) countMeasurements));
}
private static String format(final long millis) {
final long years = millis / (1000L * 3600 * 24 * 365);
final long days = millis % (1000L * 3600 * 24 * 365) / (1000 * 3600 * 24);
final long hours = (millis % (1000 * 3600 * 24)) / (1000 * 3600);
final long minutes = (millis % (1000 * 3600)) / (1000 * 60);
final long seconds = (millis % (1000 * 60)) / 1000;
final long ms = millis % 1000;
if (years > 0) {
return String.format("%d years %d days %02d:%02d:%02d,%03d", years, days, hours, minutes, seconds, ms);
} else if (days > 0) {
return String.format("%d days %02d:%02d:%02d,%03d", days, hours, minutes, seconds, ms);
}
return String.format("%02d:%02d:%02d,%03d", hours, minutes, seconds, ms);
}
private static String toBinary(final int b) {

12
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -12,7 +12,6 @@ import java.time.ZoneId;
import java.util.Optional;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
class PdbReader implements AutoCloseable {
@@ -74,7 +73,7 @@ class PdbReader implements AutoCloseable {
*/
public void seekToLastValue() {
while (readEntry(Tags.EMPTY).isPresent()) {
while (readEntry().isPresent()) {
// seek to the end
// TODO @ahr add date offsets every x kb, so we don't have
// to read the whole file
@@ -90,7 +89,7 @@ class PdbReader implements AutoCloseable {
}
}
Entry readNullableEntry(final Tags tags) throws ReadRuntimeException {
Entry readNullableEntry() throws ReadRuntimeException {
try {
final long epochMilliIncrement = readValue(ByteType.DATE_INCREMENT);
if (epochMilliIncrement < 0) {
@@ -103,15 +102,16 @@ class PdbReader implements AutoCloseable {
return null;
}
dateOffsetAtCurrentLocation = epochMilli;
return new Entry(epochMilli, value, tags);
return new Entry(epochMilli, value, pdbFile.getTags());
} catch (final IOException e) {
throw new ReadException(e);
}
}
public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
public Optional<Entry> readEntry() throws ReadRuntimeException {
final Entry entry = readNullableEntry(tags);
final Entry entry = readNullableEntry();
return Optional.ofNullable(entry);
}

39
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -1,13 +1,15 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.SortedSet;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.concurrent.BlockingQueue;
@@ -16,33 +18,30 @@ import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldNotExistsException;
import org.lucares.ludb.H2DB;
import org.lucares.ludb.Proposal;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.DataStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class PerformanceDb implements AutoCloseable, CollectionUtils {
public class PerformanceDb implements AutoCloseable {
private final static Logger LOGGER = LoggerFactory.getLogger(PerformanceDb.class);
private final static Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.ingestion.block");
private final TagsToFile tagsToFile;
private final H2DB db;
private final DataStore db;
public PerformanceDb(final Path dataDirectory) {
public PerformanceDb(final Path dataDirectory) throws IOException {
db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
db = new DataStore(dataDirectory);
tagsToFile = new TagsToFile(dataDirectory, db);
tagsToFile = new TagsToFile(db);
}
public void put(final Entry entry) throws WriteException {
@@ -192,28 +191,24 @@ public class PerformanceDb implements AutoCloseable, CollectionUtils {
@Override
public void close() {
try {
db.close();
} catch (final Exception e) {
// H2 doesn't actually do anything in close
throw new IllegalStateException(e);
}
tagsToFile.close();
}
public List<Proposal> autocomplete(final String query, final int caretIndex) {
return db.proposeTagForQuery(query, caretIndex);
// TODO implement proposals
// return db.proposeTagForQuery(query, caretIndex);
return Collections.emptyList();
}
public List<String> getFields() {
final List<Field> fields = db.getAvailableFields();
final List<String> fields = db.getAvailableFields();
return map(fields, Field::getName);
return fields;
}
public List<String> getFieldsValues(final String query, final String fieldName) throws FieldNotExistsException {
return db.getAvailableValuesForField(query, fieldName);
public SortedSet<String> getFieldsValues(final String query, final String fieldName) {
return db.getAvailableValuesForKey(query, fieldName);
}
}

78
performanceDb/src/main/java/org/lucares/performance/db/Proposal.java Normal file
View File

@@ -0,0 +1,78 @@
package org.lucares.performance.db;
public class Proposal implements Comparable<Proposal> {
private final String proposedTag;
private final String proposedQuery;
private final long results;
public Proposal(final String proposedTag, final String proposedQuery, final long results) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.results = results;
}
public String getProposedTag() {
return proposedTag;
}
public String getProposedQuery() {
return proposedQuery;
}
public long getResults() {
return results;
}
@Override
public String toString() {
return "Proposal [proposedTag=" + proposedTag + ", proposedQuery=" + proposedQuery + ", results=" + results
+ "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
result = prime * result + (int) (results ^ (results >>> 32));
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
} else if (!proposedQuery.equals(other.proposedQuery))
return false;
if (proposedTag == null) {
if (other.proposedTag != null)
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
if (results != other.results)
return false;
return true;
}
@Override
public int compareTo(final Proposal o) {
if (results != o.results) {
return results < o.results ? 1 : -1;
}
return proposedTag.compareToIgnoreCase(o.proposedTag);
}
}

19
performanceDb/src/main/java/org/lucares/performance/db/Query.java
View File

@@ -1,20 +1,27 @@
package org.lucares.performance.db;
import java.util.ArrayList;
import java.util.List;
import org.lucares.pdb.api.Tags;
final class Query {
static String createQuery(final Tags tags) {
final StringBuilder result = new StringBuilder();
final List<String> terms = new ArrayList<>();
for (final String key : tags.getKeys()) {
final String value = tags.getValue(key);
result.append(key);
result.append("=");
result.append(value);
result.append(" ");
final StringBuilder term = new StringBuilder();
term.append(key);
term.append("=");
term.append(value);
term.append(" ");
terms.add(term.toString());
}
return result.toString().trim();
return String.join(" and ", terms);
}
}

32
performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java
View File

@@ -1,32 +0,0 @@
package org.lucares.performance.db;
import java.nio.file.Path;
import java.util.UUID;
import org.lucares.pdb.api.Tags;
public class StorageUtils {
public static Path createStorageFile(final Path tagSpecificStorageFolder) {
final Path storageFile = tagSpecificStorageFolder.resolve(UUID.randomUUID().toString());
return storageFile;
}
public static Path createTagSpecificStorageFolder(final Path dataDirectory, final Tags tags) {
final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
final Path dataBaseDir = dataDirectory.resolve("data");
final Path tagSpecificFolder = dataBaseDir.resolve(tagBaseDir);
return tagSpecificFolder;
}
public static Path getTagSpecificStorageFolder(final Path storageFilePath) {
return storageFilePath //
.getParent(); // tag specific
}
}

107
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,7 +1,6 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
@@ -17,14 +16,15 @@ import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.lucares.ludb.Document;
import org.lucares.ludb.H2DB;
import org.lucares.ludb.internal.FieldNotExistsInternalException;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.DataStore;
import org.lucares.pdb.datastore.Doc;
import org.lucares.utils.CollectionUtils;
import org.lucares.utils.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TagsToFile implements CollectionUtils, AutoCloseable {
public class TagsToFile implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(TagsToFile.class);
@@ -62,31 +62,20 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
public Optional<PdbWriter> writer(final PdbFile pdbFile) {
return writers.stream().filter(w -> Objects.equals(w.getPdbFile(), pdbFile)).findAny();
}
public Optional<Path> tagSpecificBaseDir() {
if (writers.size() > 0) {
return Optional.of(writers.get(0).getPdbFile().getPath().getParent());
}
return Optional.empty();
}
}
private final H2DB db;
private final Path dataDirectory;
private final DataStore db;
private final Map<Tags, WriterCache> cachedWriters = new HashMap<>();
public TagsToFile(final Path dataDirectory, final H2DB db) {
this.dataDirectory = dataDirectory;
public TagsToFile(final DataStore db) {
this.db = db;
}
private List<PdbFile> getFilesMatchingTagsExactly(final Tags tags) {
final List<PdbFile> files = getFilesMatchingTags(tags);
return filter(files, f -> f.getTags().equals(tags));
return CollectionUtils.filter(files, f -> f.getTags().equals(tags));
}
private List<PdbFile> getFilesMatchingTags(final Tags tags) {
@@ -116,7 +105,7 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
final Tags fileSpecificTags = tagSpecific.getTags();
final List<Path> storageFiles = FileUtils.listRecursively(tagSpecific.getPath());
final List<Path> storageFiles = listFiles(tagSpecific);
for (final Path storageFile : storageFiles) {
final PdbFile pdbFile = new PdbFile(storageFile, fileSpecificTags);
@@ -128,36 +117,30 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
return result;
}
private List<Path> listFiles(final TagSpecificBaseDir tagSpecific) {
try {
return FileUtils.listRecursively(tagSpecific.getPath());
} catch (final IOException e) {
throw new ReadException(e);
}
}
private List<TagSpecificBaseDir> getTagSpecificFolders(final String query) {
final List<TagSpecificBaseDir> result = new ArrayList<>();
try {
final List<Document> searchResult = db.search(query);
for (final Document document : searchResult) {
final List<Doc> searchResult = db.search(query);
final Path path = document.getFile().toPath();
final Tags tags = toTags(document);
for (final Doc document : searchResult) {
result.add(new TagSpecificBaseDir(path, tags));
}
} catch (final FieldNotExistsInternalException e) {
// happens if there is not yet a tag specific base dir
final Path path = document.getPath();
final Tags tags = document.getTags();
result.add(new TagSpecificBaseDir(path, tags));
}
return result;
}
private Tags toTags(final Document document) {
Tags tagsOfFile = Tags.create();
for (final String key : document.getProperties().keySet()) {
final String value = document.getPropertyString(key);
tagsOfFile = tagsOfFile.copyAdd(key, value);
}
return tagsOfFile;
}
public PdbWriter getWriter(final OffsetDateTime date, final Tags tags) throws ReadException, WriteException {
final PdbWriter result;
final WriterCache writersForTags = getOrInit(tags);
@@ -173,9 +156,10 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
assertAllFilesHaveSameFolder(pdbFiles);
pdbFiles.removeIf(f -> !f.exists());
final List<Optional<PdbWriter>> optionalWriters = map(pdbFiles, writersForTags::writer);
final List<Optional<PdbWriter>> existingWriters = filter(optionalWriters, Optional::isPresent);
final List<PdbWriter> writers = map(existingWriters, Optional::get);
final List<Optional<PdbWriter>> optionalWriters = CollectionUtils.map(pdbFiles, writersForTags::writer);
final List<Optional<PdbWriter>> existingWriters = CollectionUtils.filter(optionalWriters,
Optional::isPresent);
final List<PdbWriter> writers = CollectionUtils.map(existingWriters, Optional::get);
final Optional<PdbWriter> optionalFirst = chooseBestMatchingWriter(writers, date);
@@ -233,12 +217,8 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
private PdbWriter newPdbWriter(final Tags tags) {
try {
PdbWriter result;
final Path tagSpecificStorageFolder = getOrInit(tags).tagSpecificBaseDir()
.orElse(StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags));
final PdbFile pdbFile = createNewPdbFile(tags, tagSpecificStorageFolder);
result = new PdbWriter(pdbFile);
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile);
getOrInit(tags).addWriter(result);
return result;
@@ -259,38 +239,15 @@ public class TagsToFile implements CollectionUtils, AutoCloseable {
}
}
private PdbFile createNewPdbFile(final Tags tags, final Path tagSpecificStorageFolder) throws IOException {
final Path storageFile;
PdbFile result;
storageFile = createNewFile(tagSpecificStorageFolder);
private PdbFile createNewPdbFile(final Tags tags) throws IOException {
final Document document = db.getDocument(tagSpecificStorageFolder.toFile());
if (document == null) {
db.addDocument(tagSpecificStorageFolder.toFile());
final Path storageFile = db.createNewFile(tags);
tags.forEach((fieldName, value) -> {
TagsUtils.setProperty(db, tagSpecificStorageFolder.toFile(), fieldName, value);
});
}
result = new PdbFile(storageFile, tags);
final PdbFile result = new PdbFile(storageFile, tags);
PdbWriter.init(result);
return result;
}
private Path createNewFile(final Path tagSpecificStorageFolder) {
final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder);
try {
Files.createDirectories(result.getParent());
Files.createFile(result);
} catch (final IOException e) {
throw new IllegalStateException(e); // very unlikely
}
return result;
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
for (final Entry<Tags, WriterCache> readersWriters : cachedWriters.entrySet()) {

24
performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java
View File

@@ -1,24 +0,0 @@
package org.lucares.performance.db;
import java.io.File;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldNotExistsException;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
class TagsUtils {
static void setProperty(final H2DB db, final File file, final String fieldName, final String value) {
try {
db.setProperty(file, fieldName, value);
} catch (final FieldNotExistsException e) {
db.createField(new Field(fieldName, FieldType.STRING));
try {
db.setProperty(file, fieldName, value);
} catch (final FieldNotExistsException e1) {
throw new IllegalStateException(e1);
}
}
}
}

4
performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java
View File

@@ -34,7 +34,7 @@ public class PdbReaderWriterTest {
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
org.lucares.utils.file.FileUtils.delete(dataDirectory);
}
@DataProvider(name = "providerWriteRead")
@@ -84,7 +84,7 @@ public class PdbReaderWriterTest {
for (final Entry entry : entries) {
final Entry actual = reader.readEntry(TAGS).orElseThrow(() -> new AssertionError());
final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actual, entry);
}

24
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -1,6 +1,5 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -9,13 +8,14 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.DataStore;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -33,7 +33,7 @@ public class PerformanceDbTest {
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
org.lucares.utils.file.FileUtils.delete(dataDirectory);
}
public void testInsertRead() throws Exception {
@@ -111,15 +111,19 @@ public class PerformanceDbTest {
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries);
final List<Path> foldersInStorage = Files.list(dataDirectory.resolve("data")).filter(Files::isDirectory)
.collect(Collectors.toList());
Assert.assertEquals(foldersInStorage.size(), 1);
final List<Path> filesInStorage = FileUtils.listRecursively(DataStore.storageDirectory(dataDirectory));
final Path tagSpecificFolder = foldersInStorage.get(0);
Assert.assertEquals(filesInStorage.size(), 1);
final File[] filesInStorage = tagSpecificFolder.toFile().listFiles();
Assert.assertEquals(filesInStorage.length, 1,
"one file in storage, but was: " + Arrays.asList(filesInStorage));
final Path tagSpecificFile = filesInStorage.get(0);
final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags);
try (PdbReader pdbReader = new PdbReader(pdbFile)) {
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0));
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1));
Assert.assertEquals(pdbReader.readEntry().isPresent(), false);
}
}
}

25
performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java
View File

@@ -1,25 +0,0 @@
package org.lucares.performance.db;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.lucares.pdb.api.Tags;
import org.testng.Assert;
import org.testng.annotations.Test;
@Test
public class StorageUtilsTest {
public void testGetTagSpecificStorageFolder() {
final Path dataDirectory = Paths.get("/tmp");
final Tags tags = Tags.create("key", "value");
final Path tagSpecifiStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder);
final Path extractedTagSpecifiStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFile);
Assert.assertEquals(extractedTagSpecifiStorageFolder, extractedTagSpecifiStorageFolder);
}
}

22
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -1,15 +1,14 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.DataStore;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -27,13 +26,13 @@ public class TagsToFilesTest {
@AfterMethod
public void afterMethod() throws IOException {
org.lucares.performance.db.FileUtils.delete(dataDirectory);
org.lucares.utils.file.FileUtils.delete(dataDirectory);
}
public void test() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final DataStore db = new DataStore(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue");
@@ -47,9 +46,8 @@ public class TagsToFilesTest {
}
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final DataStore db = new DataStore(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
@@ -68,8 +66,8 @@ public class TagsToFilesTest {
@Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final DataStore db = new DataStore(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db);) {
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -106,8 +104,8 @@ public class TagsToFilesTest {
public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final DataStore db = new DataStore(dataDirectory);
try (final TagsToFile tagsToFile = new TagsToFile(db)) {
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);