andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

all storage files for the same tags use the same storage folder

Browse Source 
- added an additional data folder as first level
This commit is contained in:
Andreas Huber
2016-12-23 16:35:00 +01:00
parent 85eaee940e
commit 6969c8ce46
9 changed files with 135 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
performanceDb/src/main/java/org/lucares/performance/db/Fields.java
View File

@@ -1,11 +0,0 @@
package org.lucares.performance.db;
class Fields {
static final String PREFIX_INTERNAL_TAG = "__internal_";
static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset";
static boolean isInternalField(final String key) {
return key.startsWith(PREFIX_INTERNAL_TAG);
}
}

8
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -16,8 +16,6 @@ import java.util.logging.Logger;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
import org.lucares.ludb.Proposal;
import org.lucares.pdb.api.Entry;
@@ -37,11 +35,6 @@ public class PerformanceDb implements AutoCloseable {
db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final Field dateOffsetField = db.getField(Fields.DATE_OFFSET);
if (dateOffsetField == null) {
db.createField(Fields.DATE_OFFSET, FieldType.STRING);
}
tagsToFile = new TagsToFile(dataDirectory, db);
}
@@ -103,7 +96,6 @@ public class PerformanceDb implements AutoCloseable {
final Entry entry = entryOptional.get();
final Tags tags = entry.getTags();
TagsUtils.ensureNoInternalFields(tags);
final OffsetDateTime date = entry.getDate();
final PdbWriter writer = manager.get(tags, date);

45
performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java
View File

@@ -3,14 +3,53 @@ package org.lucares.performance.db;
import java.nio.file.Path;
import java.util.UUID;
import org.lucares.pdb.api.Tags;
public class StorageUtils {
public static Path createStorageFile(final Path parent, final String tagBaseDir, final Day day) {
public static Path createStorageFile(final Path tagSpecificStorageFolder, final Day day) {
final Path tagSpecificFolder = parent.resolve(tagBaseDir);
final Path dateSpecificFolder = tagSpecificFolder.resolve(day.format("/"));
final Path dateSpecificFolder = tagSpecificStorageFolder.resolve(day.format("/"));
final Path storageFile = dateSpecificFolder.resolve(UUID.randomUUID().toString());
return storageFile;
}
public static Day getDateOffset(final Path pathToStorageFile) {
try {
final Path pathDay = pathToStorageFile.getParent();
final Path pathMonth = pathDay.getParent();
final Path pathYear = pathMonth.getParent();
final int day = Integer.parseInt(pathDay.getFileName().toString(), 10);
final int month = Integer.parseInt(pathMonth.getFileName().toString(), 10);
final int year = Integer.parseInt(pathYear.getFileName().toString(), 10);
final Day result = new Day(year, month, day);
return result;
} catch (final NumberFormatException e) {
throw new IllegalStateException(pathToStorageFile.toUri().getPath() + " is not a path to a storage file",
e);
}
}
public static Path createTagSpecificStorageFolder(final Path dataDirectory, final Tags tags) {
final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
final Path dataBaseDir = dataDirectory.resolve("data");
final Path tagSpecificFolder = dataBaseDir.resolve(tagBaseDir);
return tagSpecificFolder;
}
public static Path getTagSpecificStorageFolder(final Path storageFilePath) {
return storageFilePath //
.getParent() // day
.getParent() // month
.getParent() // year
.getParent(); // tag specific
}
}

71
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -9,12 +9,10 @@ import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.Set;
import java.util.stream.Collectors;
import org.lucares.ludb.Document;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Tags;
@@ -60,8 +58,7 @@ public class TagsToFile implements CollectionUtils {
private PdbFile toPdbFile(final Document document) {
final File file = document.getFile();
final String dateOffset = document.getPropertyString(Fields.DATE_OFFSET);
final Day day = Day.fromString(dateOffset);
final Day day = StorageUtils.getDateOffset(file.toPath());
final Tags tagsOfFile = toTags(document);
final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile);
return pdbFile;
@@ -72,10 +69,8 @@ public class TagsToFile implements CollectionUtils {
for (final String key : document.getProperties().keySet()) {
if (!Fields.isInternalField(key)) {
final String value = document.getPropertyString(key);
tagsOfFile = tagsOfFile.copyAdd(key, value);
}
final String value = document.getPropertyString(key);
tagsOfFile = tagsOfFile.copyAdd(key, value);
}
return tagsOfFile;
}
@@ -85,8 +80,12 @@ public class TagsToFile implements CollectionUtils {
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
final List<PdbFile> preResult = new ArrayList<>();
assertAllFilesHaveSameFolder(pdbFiles);
PdbFile result;
for (final PdbFile pdbFile : pdbFiles) {
// TODO @ahr should compare with the last written date
final boolean inRange = pdbFile.getTimeRange().inRange(date);
if (inRange) {
@@ -99,55 +98,55 @@ public class TagsToFile implements CollectionUtils {
}
if (preResult.isEmpty()) {
result = createNewPdbFile(date, tags);
} else {
if (preResult.size() != 1) {
throw new IllegalStateException("found not exactly one matching file: " + preResult);
Path tagSpecificStorageFolder;
if (pdbFiles.isEmpty()) {
tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
} else {
final Path storageFilePath = pdbFiles.get(0).getFile().toPath();
tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
}
result = createNewPdbFile(date, tags, tagSpecificStorageFolder);
} else {
result = preResult.get(0);
}
return result;
}
private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) {
private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
final Set<File> reducedFolder = pdbFiles.stream()//
.map(PdbFile::getFile)//
.map(f -> f.getParentFile().getParentFile().getParentFile())//
.collect(Collectors.toSet());
if (reducedFolder.size() > 1) {
throw new IllegalStateException(
"All storage folders for the same tag must be the same, but are not: " + reducedFolder);
}
}
private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) {
final File file;
PdbFile result;
file = createNewFile(date, tags);
file = createNewFile(date, tagSpecificStorageFolder);
final Day day = new Day(date);
db.addDocument(file);
ensureFieldsExist(tags);
tags.forEach((fieldName, value) -> {
TagsUtils.setProperty(db, file, fieldName, value);
});
TagsUtils.setProperty(db, file, Fields.DATE_OFFSET, day.serialize());
result = new PdbFile(day, file, tags);
return result;
}
private void ensureFieldsExist(final Tags tags) {
final List<Field> fields = db.getAvailableFields("");
final Map<String, Field> fieldsMap = toMap(fields, Field::getName);
tags.forEach((key, value) -> {
final Field field = fieldsMap.get(key);
if (field == null) {
db.createField(key, FieldType.STRING);
}
});
}
private File createNewFile(final OffsetDateTime date, final Tags tags) {
private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
final Day day = new Day(date);
final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
final Path result = StorageUtils.createStorageFile(dataDirectory, tagBaseDir, day);
final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
try {
Files.createDirectories(result.getParent());
Files.createFile(result);

8
performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java
View File

@@ -5,16 +5,8 @@ import java.io.File;
import org.lucares.ludb.FieldNotExistsException;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Tags;
class TagsUtils {
static void ensureNoInternalFields(final Tags tags) {
tags.getKeys().forEach(key -> {
if (Fields.isInternalField(key)) {
throw new IllegalArgumentException(key + " is an internal field. Choose another prefix.");
}
});
}
static void setProperty(final H2DB db, final File file, final String fieldName, final String value) {
try {

6
performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java
View File

@@ -277,10 +277,10 @@ public class ObjectMapperTest {
private void fill(final TagDB tagDB) {
final List<String> pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "vapondem" + i)
final List<String> pods = IntStream.rangeClosed(1, 1).mapToObj(i -> "vapondem" + i)
.collect(Collectors.toList());
final List<String> hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList());
final List<String> versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList());
final List<String> hosts = IntStream.rangeClosed(1, 1).mapToObj(i -> "host" + i).collect(Collectors.toList());
final List<String> versions = IntStream.rangeClosed(1, 1).mapToObj(i -> "5." + i).collect(Collectors.toList());
final List<String> types = Arrays.asList("app", "engine", "web", "batch");
for (final String pod : pods) {

2
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -111,7 +111,7 @@ public class PerformanceDbTest {
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries);
final List<Path> foldersInStorage = Files.list(dataDirectory).filter(Files::isDirectory)
final List<Path> foldersInStorage = Files.list(dataDirectory.resolve("data")).filter(Files::isDirectory)
.collect(Collectors.toList());
Assert.assertEquals(foldersInStorage.size(), 1);

39
performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java Normal file
View File

@@ -0,0 +1,39 @@
package org.lucares.performance.db;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.lucares.pdb.api.Tags;
import org.testng.Assert;
import org.testng.annotations.Test;
@Test
public class StorageUtilsTest {
public void testExtractDateOffsetFromPath() throws Exception {
final Path tagSpecificStorageFolder = Paths.get("/tmp");
final Day day = new Day(2016, 1, 1);
final Path storageFile = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
final Day extractedDateOffset = StorageUtils.getDateOffset(storageFile);
Assert.assertEquals(extractedDateOffset, day);
}
public void testGetTagSpecificStorageFolder() {
final Path dataDirectory = Paths.get("/tmp");
final Tags tags = Tags.create("key", "value");
final Day day = new Day(2016, 1, 1);
final Path tagSpecifiStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder, day);
final Path extractedTagSpecifiStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFile);
Assert.assertEquals(extractedTagSpecifiStorageFolder, extractedTagSpecifiStorageFolder);
}
}

22
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -7,7 +7,6 @@ import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
@@ -35,8 +34,6 @@ public class TagsToFilesTest {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
db.createField(Fields.DATE_OFFSET, FieldType.STRING);
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
@@ -46,14 +43,12 @@ public class TagsToFilesTest {
final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
Assert.assertEquals(newFileForTags, existingFileForTags);
}
}
public void testMultipleFilesForTag() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
db.createField(Fields.DATE_OFFSET, FieldType.STRING);
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
@@ -75,12 +70,13 @@ public class TagsToFilesTest {
public void testNewFileIfDateIsTooOld() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
db.createField(Fields.DATE_OFFSET, FieldType.STRING);
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
final OffsetDateTime earlyMorning = DateUtils.getDate(2016, 1, 1, 8, 1, 1);
final OffsetDateTime evening = DateUtils.getDate(2016, 1, 1, 18, 1, 1);
final Tags tags = Tags.create("myKey", "myValue");
@@ -88,15 +84,27 @@ public class TagsToFilesTest {
PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags));
final PdbFile fileMorning = tagsToFile.getFile(morning, tags);
PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
Assert.assertNotEquals(fileAfternoon, fileMorning);
final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags);
PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
Assert.assertNotEquals(fileEarlyMorning, fileAfternoon);
Assert.assertNotEquals(fileEarlyMorning, fileMorning);
final PdbFile fileEvening = tagsToFile.getFile(evening, tags);
Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file");
Assert.assertNotEquals(fileEvening, fileMorning);
Assert.assertNotEquals(fileEvening, fileEarlyMorning);
}
}
public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
db.createField(Fields.DATE_OFFSET, FieldType.STRING);
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);