all storage files for the same tags use the same storage folder

- added an additional data folder as first level

performanceDb/src/main/java/org/lucares/performance/db/Fields.java

@@ -1,11 +0,0 @@
-package org.lucares.performance.db;
-
-class Fields {
-	static final String PREFIX_INTERNAL_TAG = "__internal_";
-
-	static final String DATE_OFFSET = PREFIX_INTERNAL_TAG + "date_offset";
-
-	static boolean isInternalField(final String key) {
-		return key.startsWith(PREFIX_INTERNAL_TAG);
-	}
-}

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -16,8 +16,6 @@ import java.util.logging.Logger;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
-import org.lucares.ludb.Field;
-import org.lucares.ludb.FieldType;
 import org.lucares.ludb.H2DB;
 import org.lucares.ludb.Proposal;
 import org.lucares.pdb.api.Entry;
@@ -37,11 +35,6 @@ public class PerformanceDb implements AutoCloseable {
 
 		db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
 
-		final Field dateOffsetField = db.getField(Fields.DATE_OFFSET);
-		if (dateOffsetField == null) {
-			db.createField(Fields.DATE_OFFSET, FieldType.STRING);
-		}
-
 		tagsToFile = new TagsToFile(dataDirectory, db);
 	}
 
@@ -103,7 +96,6 @@ public class PerformanceDb implements AutoCloseable {
 				final Entry entry = entryOptional.get();
 
 				final Tags tags = entry.getTags();
-				TagsUtils.ensureNoInternalFields(tags);
 				final OffsetDateTime date = entry.getDate();
 
 				final PdbWriter writer = manager.get(tags, date);

performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java

@@ -3,14 +3,53 @@ package org.lucares.performance.db;
 import java.nio.file.Path;
 import java.util.UUID;
 
+import org.lucares.pdb.api.Tags;
+
 public class StorageUtils {
 
-	public static Path createStorageFile(final Path parent, final String tagBaseDir, final Day day) {
+	public static Path createStorageFile(final Path tagSpecificStorageFolder, final Day day) {
 
-		final Path tagSpecificFolder = parent.resolve(tagBaseDir);
+		final Path dateSpecificFolder = tagSpecificStorageFolder.resolve(day.format("/"));
-		final Path dateSpecificFolder = tagSpecificFolder.resolve(day.format("/"));
 		final Path storageFile = dateSpecificFolder.resolve(UUID.randomUUID().toString());
 
 		return storageFile;
 	}
+
+	public static Day getDateOffset(final Path pathToStorageFile) {
+
+		try {
+			final Path pathDay = pathToStorageFile.getParent();
+			final Path pathMonth = pathDay.getParent();
+			final Path pathYear = pathMonth.getParent();
+
+			final int day = Integer.parseInt(pathDay.getFileName().toString(), 10);
+			final int month = Integer.parseInt(pathMonth.getFileName().toString(), 10);
+			final int year = Integer.parseInt(pathYear.getFileName().toString(), 10);
+
+			final Day result = new Day(year, month, day);
+			return result;
+		} catch (final NumberFormatException e) {
+			throw new IllegalStateException(pathToStorageFile.toUri().getPath() + " is not a path to a storage file",
+					e);
+		}
+	}
+
+	public static Path createTagSpecificStorageFolder(final Path dataDirectory, final Tags tags) {
+
+		final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
+
+		final Path dataBaseDir = dataDirectory.resolve("data");
+		final Path tagSpecificFolder = dataBaseDir.resolve(tagBaseDir);
+
+		return tagSpecificFolder;
+	}
+
+	public static Path getTagSpecificStorageFolder(final Path storageFilePath) {
+
+		return storageFilePath //
+				.getParent() // day
+				.getParent() // month
+				.getParent() // year
+				.getParent(); // tag specific
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -9,12 +9,10 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
+import java.util.Set;
-import java.util.UUID;
+import java.util.stream.Collectors;
 
 import org.lucares.ludb.Document;
-import org.lucares.ludb.Field;
-import org.lucares.ludb.FieldType;
 import org.lucares.ludb.H2DB;
 import org.lucares.pdb.api.Tags;
 
@@ -60,8 +58,7 @@ public class TagsToFile implements CollectionUtils {
 
 	private PdbFile toPdbFile(final Document document) {
 		final File file = document.getFile();
-		final String dateOffset = document.getPropertyString(Fields.DATE_OFFSET);
+		final Day day = StorageUtils.getDateOffset(file.toPath());
-		final Day day = Day.fromString(dateOffset);
 		final Tags tagsOfFile = toTags(document);
 		final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile);
 		return pdbFile;
@@ -72,10 +69,8 @@ public class TagsToFile implements CollectionUtils {
 
 		for (final String key : document.getProperties().keySet()) {
 
-			if (!Fields.isInternalField(key)) {
+			final String value = document.getPropertyString(key);
-				final String value = document.getPropertyString(key);
+			tagsOfFile = tagsOfFile.copyAdd(key, value);
-				tagsOfFile = tagsOfFile.copyAdd(key, value);
-			}
 		}
 		return tagsOfFile;
 	}
@@ -85,8 +80,12 @@ public class TagsToFile implements CollectionUtils {
 		final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
 		final List<PdbFile> preResult = new ArrayList<>();
 
+		assertAllFilesHaveSameFolder(pdbFiles);
+
 		PdbFile result;
 		for (final PdbFile pdbFile : pdbFiles) {
+
+			// TODO @ahr should compare with the last written date
 			final boolean inRange = pdbFile.getTimeRange().inRange(date);
 
 			if (inRange) {
@@ -99,55 +98,55 @@ public class TagsToFile implements CollectionUtils {
 		}
 
 		if (preResult.isEmpty()) {
-			result = createNewPdbFile(date, tags);
+
-		} else {
+			Path tagSpecificStorageFolder;
-			if (preResult.size() != 1) {
+			if (pdbFiles.isEmpty()) {
-				throw new IllegalStateException("found not exactly one matching file: " + preResult);
+				tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
+			} else {
+				final Path storageFilePath = pdbFiles.get(0).getFile().toPath();
+				tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
 			}
+
+			result = createNewPdbFile(date, tags, tagSpecificStorageFolder);
+		} else {
 			result = preResult.get(0);
 		}
 
 		return result;
 	}
 
-	private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) {
+	private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
+		final Set<File> reducedFolder = pdbFiles.stream()//
+				.map(PdbFile::getFile)//
+				.map(f -> f.getParentFile().getParentFile().getParentFile())//
+				.collect(Collectors.toSet());
+
+		if (reducedFolder.size() > 1) {
+			throw new IllegalStateException(
+					"All storage folders for the same tag must be the same, but are not: " + reducedFolder);
+		}
+	}
+
+	private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) {
 		final File file;
 		PdbFile result;
-		file = createNewFile(date, tags);
+		file = createNewFile(date, tagSpecificStorageFolder);
 		final Day day = new Day(date);
 
 		db.addDocument(file);
 
-		ensureFieldsExist(tags);
-
 		tags.forEach((fieldName, value) -> {
 			TagsUtils.setProperty(db, file, fieldName, value);
 		});
 
-		TagsUtils.setProperty(db, file, Fields.DATE_OFFSET, day.serialize());
-
 		result = new PdbFile(day, file, tags);
 		return result;
 	}
 
-	private void ensureFieldsExist(final Tags tags) {
+	private File createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
-		final List<Field> fields = db.getAvailableFields("");
-		final Map<String, Field> fieldsMap = toMap(fields, Field::getName);
-
-		tags.forEach((key, value) -> {
-
-			final Field field = fieldsMap.get(key);
-			if (field == null) {
-				db.createField(key, FieldType.STRING);
-			}
-		});
-	}
-
-	private File createNewFile(final OffsetDateTime date, final Tags tags) {
 		final Day day = new Day(date);
-		final String tagBaseDir = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
 
-		final Path result = StorageUtils.createStorageFile(dataDirectory, tagBaseDir, day);
+		final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
 		try {
 			Files.createDirectories(result.getParent());
 			Files.createFile(result);

performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java

@@ -5,16 +5,8 @@ import java.io.File;
 import org.lucares.ludb.FieldNotExistsException;
 import org.lucares.ludb.FieldType;
 import org.lucares.ludb.H2DB;
-import org.lucares.pdb.api.Tags;
 
 class TagsUtils {
-	static void ensureNoInternalFields(final Tags tags) {
-		tags.getKeys().forEach(key -> {
-			if (Fields.isInternalField(key)) {
-				throw new IllegalArgumentException(key + " is an internal field. Choose another prefix.");
-			}
-		});
-	}
 
 	static void setProperty(final H2DB db, final File file, final String fieldName, final String value) {
 		try {

performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java

@@ -277,10 +277,10 @@ public class ObjectMapperTest {
 
 	private void fill(final TagDB tagDB) {
 
-		final List<String> pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "vapondem" + i)
+		final List<String> pods = IntStream.rangeClosed(1, 1).mapToObj(i -> "vapondem" + i)
 				.collect(Collectors.toList());
-		final List<String> hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList());
+		final List<String> hosts = IntStream.rangeClosed(1, 1).mapToObj(i -> "host" + i).collect(Collectors.toList());
-		final List<String> versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList());
+		final List<String> versions = IntStream.rangeClosed(1, 1).mapToObj(i -> "5." + i).collect(Collectors.toList());
 		final List<String> types = Arrays.asList("app", "engine", "web", "batch");
 
 		for (final String pod : pods) {

performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java

@@ -111,7 +111,7 @@ public class PerformanceDbTest {
 			final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
 			Assert.assertEquals(actualEntries, entries);
 
-			final List<Path> foldersInStorage = Files.list(dataDirectory).filter(Files::isDirectory)
+			final List<Path> foldersInStorage = Files.list(dataDirectory.resolve("data")).filter(Files::isDirectory)
 					.collect(Collectors.toList());
 			Assert.assertEquals(foldersInStorage.size(), 1);
 

performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java

@@ -0,0 +1,39 @@
+package org.lucares.performance.db;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.lucares.pdb.api.Tags;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+@Test
+public class StorageUtilsTest {
+
+	public void testExtractDateOffsetFromPath() throws Exception {
+
+		final Path tagSpecificStorageFolder = Paths.get("/tmp");
+		final Day day = new Day(2016, 1, 1);
+
+		final Path storageFile = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
+
+		final Day extractedDateOffset = StorageUtils.getDateOffset(storageFile);
+
+		Assert.assertEquals(extractedDateOffset, day);
+	}
+
+	public void testGetTagSpecificStorageFolder() {
+		final Path dataDirectory = Paths.get("/tmp");
+		final Tags tags = Tags.create("key", "value");
+		final Day day = new Day(2016, 1, 1);
+
+		final Path tagSpecifiStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
+
+		final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder, day);
+
+		final Path extractedTagSpecifiStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFile);
+
+		Assert.assertEquals(extractedTagSpecifiStorageFolder, extractedTagSpecifiStorageFolder);
+	}
+
+}

performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java

@@ -7,7 +7,6 @@ import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 
-import org.lucares.ludb.FieldType;
 import org.lucares.ludb.H2DB;
 import org.lucares.pdb.api.Entry;
 import org.lucares.pdb.api.Tags;
@@ -35,8 +34,6 @@ public class TagsToFilesTest {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
 
-			db.createField(Fields.DATE_OFFSET, FieldType.STRING);
-
 			final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
 
 			final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
@@ -46,14 +43,12 @@ public class TagsToFilesTest {
 			final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
 
 			Assert.assertEquals(newFileForTags, existingFileForTags);
-
 		}
 	}
 
 	public void testMultipleFilesForTag() throws Exception {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
-			db.createField(Fields.DATE_OFFSET, FieldType.STRING);
 
 			final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
 
@@ -75,12 +70,13 @@ public class TagsToFilesTest {
 	public void testNewFileIfDateIsTooOld() throws Exception {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
-			db.createField(Fields.DATE_OFFSET, FieldType.STRING);
 
 			final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
 
 			final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
 			final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
+			final OffsetDateTime earlyMorning = DateUtils.getDate(2016, 1, 1, 8, 1, 1);
+			final OffsetDateTime evening = DateUtils.getDate(2016, 1, 1, 18, 1, 1);
 
 			final Tags tags = Tags.create("myKey", "myValue");
 
@@ -88,15 +84,27 @@ public class TagsToFilesTest {
 			PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags));
 
 			final PdbFile fileMorning = tagsToFile.getFile(morning, tags);
+			PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
 
 			Assert.assertNotEquals(fileAfternoon, fileMorning);
+
+			final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags);
+			PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
+
+			Assert.assertNotEquals(fileEarlyMorning, fileAfternoon);
+			Assert.assertNotEquals(fileEarlyMorning, fileMorning);
+
+			final PdbFile fileEvening = tagsToFile.getFile(evening, tags);
+
+			Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file");
+			Assert.assertNotEquals(fileEvening, fileMorning);
+			Assert.assertNotEquals(fileEvening, fileEarlyMorning);
 		}
 	}
 
 	public void testIdenticalDatesGoIntoSameFile() throws Exception {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
-			db.createField(Fields.DATE_OFFSET, FieldType.STRING);
 
 			final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);