diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Fields.java b/performanceDb/src/main/java/org/lucares/performance/db/Fields.java deleted file mode 100644 index 2019c20..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/Fields.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.lucares.performance.db; - -class Fields { - static final String PREFIX_INTERNAL_TAG = "__internal_"; - - static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset"; - - static boolean isInternalField(final String key) { - return key.startsWith(PREFIX_INTERNAL_TAG); - } -} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index 7f8faed..5273426 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -16,8 +16,6 @@ import java.util.logging.Logger; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import org.lucares.ludb.Field; -import org.lucares.ludb.FieldType; import org.lucares.ludb.H2DB; import org.lucares.ludb.Proposal; import org.lucares.pdb.api.Entry; @@ -37,11 +35,6 @@ public class PerformanceDb implements AutoCloseable { db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); - final Field dateOffsetField = db.getField(Fields.DATE_OFFSET); - if (dateOffsetField == null) { - db.createField(Fields.DATE_OFFSET, FieldType.STRING); - } - tagsToFile = new TagsToFile(dataDirectory, db); } @@ -103,7 +96,6 @@ public class PerformanceDb implements AutoCloseable { final Entry entry = entryOptional.get(); final Tags tags = entry.getTags(); - TagsUtils.ensureNoInternalFields(tags); final OffsetDateTime date = entry.getDate(); final PdbWriter writer = manager.get(tags, date); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java index 1541739..1e47d78 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java @@ -3,14 +3,53 @@ package org.lucares.performance.db; import java.nio.file.Path; import java.util.UUID; +import org.lucares.pdb.api.Tags; + public class StorageUtils { - public static Path createStorageFile(final Path parent, final String tagBaseDir, final Day day) { + public static Path createStorageFile(final Path tagSpecificStorageFolder, final Day day) { - final Path tagSpecificFolder = parent.resolve(tagBaseDir); - final Path dateSpecificFolder = tagSpecificFolder.resolve(day.format("/")); + final Path dateSpecificFolder = tagSpecificStorageFolder.resolve(day.format("/")); final Path storageFile = dateSpecificFolder.resolve(UUID.randomUUID().toString()); return storageFile; } + + public static Day getDateOffset(final Path pathToStorageFile) { + + try { + final Path pathDay = pathToStorageFile.getParent(); + final Path pathMonth = pathDay.getParent(); + final Path pathYear = pathMonth.getParent(); + + final int day = Integer.parseInt(pathDay.getFileName().toString(), 10); + final int month = Integer.parseInt(pathMonth.getFileName().toString(), 10); + final int year = Integer.parseInt(pathYear.getFileName().toString(), 10); + + final Day result = new Day(year, month, day); + return result; + } catch (final NumberFormatException e) { + throw new IllegalStateException(pathToStorageFile.toUri().getPath() + " is not a path to a storage file", + e); + } + } + + public static Path createTagSpecificStorageFolder(final Path dataDirectory, final Tags tags) { + + final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString(); + + final Path dataBaseDir = dataDirectory.resolve("data"); + final Path tagSpecificFolder = dataBaseDir.resolve(tagBaseDir); + + return tagSpecificFolder; + } + + public static Path getTagSpecificStorageFolder(final Path storageFilePath) { + + return storageFilePath // + .getParent() // day + .getParent() // month + .getParent() // year + .getParent(); // tag specific + } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index edf4cb0..98a7003 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -9,12 +9,10 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.Set; +import java.util.stream.Collectors; import org.lucares.ludb.Document; -import org.lucares.ludb.Field; -import org.lucares.ludb.FieldType; import org.lucares.ludb.H2DB; import org.lucares.pdb.api.Tags; @@ -60,8 +58,7 @@ public class TagsToFile implements CollectionUtils { private PdbFile toPdbFile(final Document document) { final File file = document.getFile(); - final String dateOffset = document.getPropertyString(Fields.DATE_OFFSET); - final Day day = Day.fromString(dateOffset); + final Day day = StorageUtils.getDateOffset(file.toPath()); final Tags tagsOfFile = toTags(document); final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile); return pdbFile; @@ -72,10 +69,8 @@ public class TagsToFile implements CollectionUtils { for (final String key : document.getProperties().keySet()) { - if (!Fields.isInternalField(key)) { - final String value = document.getPropertyString(key); - tagsOfFile = tagsOfFile.copyAdd(key, value); - } + final String value = document.getPropertyString(key); + tagsOfFile = tagsOfFile.copyAdd(key, value); } return tagsOfFile; } @@ -85,8 +80,12 @@ public class TagsToFile implements CollectionUtils { final List pdbFiles = getFilesMatchingTagsExactly(tags); final List preResult = new ArrayList<>(); + assertAllFilesHaveSameFolder(pdbFiles); + PdbFile result; for (final PdbFile pdbFile : pdbFiles) { + + // TODO @ahr should compare with the last written date final boolean inRange = pdbFile.getTimeRange().inRange(date); if (inRange) { @@ -99,55 +98,55 @@ public class TagsToFile implements CollectionUtils { } if (preResult.isEmpty()) { - result = createNewPdbFile(date, tags); - } else { - if (preResult.size() != 1) { - throw new IllegalStateException("found not exactly one matching file: " + preResult); + + Path tagSpecificStorageFolder; + if (pdbFiles.isEmpty()) { + tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags); + } else { + final Path storageFilePath = pdbFiles.get(0).getFile().toPath(); + tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath); } + + result = createNewPdbFile(date, tags, tagSpecificStorageFolder); + } else { result = preResult.get(0); } return result; } - private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) { + private void assertAllFilesHaveSameFolder(final List pdbFiles) { + final Set reducedFolder = pdbFiles.stream()// + .map(PdbFile::getFile)// + .map(f -> f.getParentFile().getParentFile().getParentFile())// + .collect(Collectors.toSet()); + + if (reducedFolder.size() > 1) { + throw new IllegalStateException( + "All storage folders for the same tag must be the same, but are not: " + reducedFolder); + } + } + + private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) { final File file; PdbFile result; - file = createNewFile(date, tags); + file = createNewFile(date, tagSpecificStorageFolder); final Day day = new Day(date); db.addDocument(file); - ensureFieldsExist(tags); - tags.forEach((fieldName, value) -> { TagsUtils.setProperty(db, file, fieldName, value); }); - TagsUtils.setProperty(db, file, Fields.DATE_OFFSET, day.serialize()); - result = new PdbFile(day, file, tags); return result; } - private void ensureFieldsExist(final Tags tags) { - final List fields = db.getAvailableFields(""); - final Map fieldsMap = toMap(fields, Field::getName); - - tags.forEach((key, value) -> { - - final Field field = fieldsMap.get(key); - if (field == null) { - db.createField(key, FieldType.STRING); - } - }); - } - - private File createNewFile(final OffsetDateTime date, final Tags tags) { + private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) { final Day day = new Day(date); - final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString(); - final Path result = StorageUtils.createStorageFile(dataDirectory, tagBaseDir, day); + final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day); try { Files.createDirectories(result.getParent()); Files.createFile(result); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java index ecad3e3..219d028 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java @@ -5,16 +5,8 @@ import java.io.File; import org.lucares.ludb.FieldNotExistsException; import org.lucares.ludb.FieldType; import org.lucares.ludb.H2DB; -import org.lucares.pdb.api.Tags; class TagsUtils { - static void ensureNoInternalFields(final Tags tags) { - tags.getKeys().forEach(key -> { - if (Fields.isInternalField(key)) { - throw new IllegalArgumentException(key + " is an internal field. Choose another prefix."); - } - }); - } static void setProperty(final H2DB db, final File file, final String fieldName, final String value) { try { diff --git a/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java b/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java index 2044288..992bdea 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java @@ -277,10 +277,10 @@ public class ObjectMapperTest { private void fill(final TagDB tagDB) { - final List pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "vapondem" + i) + final List pods = IntStream.rangeClosed(1, 1).mapToObj(i -> "vapondem" + i) .collect(Collectors.toList()); - final List hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList()); - final List versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList()); + final List hosts = IntStream.rangeClosed(1, 1).mapToObj(i -> "host" + i).collect(Collectors.toList()); + final List versions = IntStream.rangeClosed(1, 1).mapToObj(i -> "5." + i).collect(Collectors.toList()); final List types = Arrays.asList("app", "engine", "web", "batch"); for (final String pod : pods) { diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java index 16b14b1..d9a0c20 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -111,7 +111,7 @@ public class PerformanceDbTest { final List actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList(); Assert.assertEquals(actualEntries, entries); - final List foldersInStorage = Files.list(dataDirectory).filter(Files::isDirectory) + final List foldersInStorage = Files.list(dataDirectory.resolve("data")).filter(Files::isDirectory) .collect(Collectors.toList()); Assert.assertEquals(foldersInStorage.size(), 1); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java b/performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java new file mode 100644 index 0000000..ca97869 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java @@ -0,0 +1,39 @@ +package org.lucares.performance.db; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.lucares.pdb.api.Tags; +import org.testng.Assert; +import org.testng.annotations.Test; + +@Test +public class StorageUtilsTest { + + public void testExtractDateOffsetFromPath() throws Exception { + + final Path tagSpecificStorageFolder = Paths.get("/tmp"); + final Day day = new Day(2016, 1, 1); + + final Path storageFile = StorageUtils.createStorageFile(tagSpecificStorageFolder, day); + + final Day extractedDateOffset = StorageUtils.getDateOffset(storageFile); + + Assert.assertEquals(extractedDateOffset, day); + } + + public void testGetTagSpecificStorageFolder() { + final Path dataDirectory = Paths.get("/tmp"); + final Tags tags = Tags.create("key", "value"); + final Day day = new Day(2016, 1, 1); + + final Path tagSpecifiStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags); + + final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder, day); + + final Path extractedTagSpecifiStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFile); + + Assert.assertEquals(extractedTagSpecifiStorageFolder, extractedTagSpecifiStorageFolder); + } + +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java index 782901b..fd38139 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -7,7 +7,6 @@ import java.nio.file.Path; import java.time.OffsetDateTime; import java.time.ZoneOffset; -import org.lucares.ludb.FieldType; import org.lucares.ludb.H2DB; import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Tags; @@ -35,8 +34,6 @@ public class TagsToFilesTest { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - db.createField(Fields.DATE_OFFSET, FieldType.STRING); - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); @@ -46,14 +43,12 @@ public class TagsToFilesTest { final PdbFile existingFileForTags = tagsToFile.getFile(date, tags); Assert.assertEquals(newFileForTags, existingFileForTags); - } } public void testMultipleFilesForTag() throws Exception { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - db.createField(Fields.DATE_OFFSET, FieldType.STRING); final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); @@ -75,12 +70,13 @@ public class TagsToFilesTest { public void testNewFileIfDateIsTooOld() throws Exception { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - db.createField(Fields.DATE_OFFSET, FieldType.STRING); final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1); + final OffsetDateTime earlyMorning = DateUtils.getDate(2016, 1, 1, 8, 1, 1); + final OffsetDateTime evening = DateUtils.getDate(2016, 1, 1, 18, 1, 1); final Tags tags = Tags.create("myKey", "myValue"); @@ -88,15 +84,27 @@ public class TagsToFilesTest { PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags)); final PdbFile fileMorning = tagsToFile.getFile(morning, tags); + PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags)); Assert.assertNotEquals(fileAfternoon, fileMorning); + + final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags); + PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags)); + + Assert.assertNotEquals(fileEarlyMorning, fileAfternoon); + Assert.assertNotEquals(fileEarlyMorning, fileMorning); + + final PdbFile fileEvening = tagsToFile.getFile(evening, tags); + + Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file"); + Assert.assertNotEquals(fileEvening, fileMorning); + Assert.assertNotEquals(fileEvening, fileEarlyMorning); } } public void testIdenticalDatesGoIntoSameFile() throws Exception { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - db.createField(Fields.DATE_OFFSET, FieldType.STRING); final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);