insert entries for different tags in one stream

performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java

@@ -3,7 +3,7 @@ package org.lucares.performance.db;
 import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 
-final class BlockingQueueIterator<E> implements BlockingIterator<E> {
+public final class BlockingQueueIterator<E> implements BlockingIterator<E> {
 
 	private final BlockingQueue<E> queue;
 

performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java

@@ -1,11 +1,14 @@
 package org.lucares.performance.db;
 
-import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalQuery;
 import java.util.Calendar;
-import java.util.Date;
 import java.util.TimeZone;
 
 public class DateUtils {
@@ -33,14 +36,22 @@ public class DateUtils {
 		return result;
 	}
 
-	public static String format(final Date date) {
-		final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH-mm:ss,SSS Z");
-		dateFormat.setTimeZone(TIME_ZONE_UTC);
-
-		return dateFormat.format(date);
-	}
-
 	public static OffsetDateTime nowInUtc() {
 		return OffsetDateTime.now(ZoneOffset.UTC);
 	}
+
+	public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter,
+			final ZoneOffset zoneOffset) {
+		final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
+
+			@Override
+			public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
+				final LocalDate localDate = LocalDate.from(temporal);
+				final LocalTime localTime = LocalTime.from(temporal);
+				return OffsetDateTime.of(localDate, localTime, zoneOffset);
+			}
+		};
+		final OffsetDateTime date = formatter.parse(text, query);
+		return date;
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/Entry.java

@@ -9,20 +9,40 @@ import java.util.Comparator;
 public class Entry {
 	public static final Comparator<Entry> BY_DATE = new EntryByDateComparator();
 
+	/**
+	 * A special {@link Entry} that can be used as poison object for
+	 * {@link BlockingQueueIterator}.
+	 */
+	public static final Entry POISON = new Entry(0, -1);
+
+	public static final long MAX_VALUE = 0xFF_FF_FF_FFL;
+
 	private final long epochMilli;
 
 	private final long value;
 
-	public Entry(final OffsetDateTime date, final long value) {
+	private final Tags tags;
-		super();
+
+	public Entry(final OffsetDateTime date, final long value, final Tags tags) {
+		this.tags = tags;
 		this.epochMilli = date.toInstant().toEpochMilli();
 		this.value = value;
 	}
 
-	Entry(final long epochMilli, final long value) {
+	Entry(final long epochMilli, final long value, final Tags tags) {
-		super();
+		if (value < 0 || value > MAX_VALUE) {
+			throw new IllegalArgumentException("value must be between 0 and " + MAX_VALUE);
+		}
+
 		this.epochMilli = epochMilli;
 		this.value = value;
+		this.tags = tags;
+	}
+
+	private Entry(final long epochMilli, final long value) {
+		this.epochMilli = epochMilli;
+		this.value = value;
+		this.tags = null;
 	}
 
 	public OffsetDateTime getDate() {
@@ -38,10 +58,14 @@ public class Entry {
 		return epochMilli;
 	}
 
+	public Tags getTags() {
+		return tags;
+	}
+
 	@Override
 	public String toString() {
 		final OffsetDateTime date = getDate();
-		return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value;
+		return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags + ")";
 	}
 
 	@Override
@@ -49,6 +73,7 @@ public class Entry {
 		final int prime = 31;
 		int result = 1;
 		result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32));
+		result = prime * result + ((tags == null) ? 0 : tags.hashCode());
 		result = prime * result + (int) (value ^ (value >>> 32));
 		return result;
 	}
@@ -64,9 +89,13 @@ public class Entry {
 		final Entry other = (Entry) obj;
 		if (epochMilli != other.epochMilli)
 			return false;
+		if (tags == null) {
+			if (other.tags != null)
+				return false;
+		} else if (!tags.equals(other.tags))
+			return false;
 		if (value != other.value)
 			return false;
 		return true;
 	}
-
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java

@@ -5,6 +5,8 @@ import java.util.Comparator;
 
 public class PdbFileByTimeAsc implements Comparator<PdbFile> {
 
+	public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc();
+
 	@Override
 	public int compare(final PdbFile o1, final PdbFile o2) {
 

performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java

@@ -18,6 +18,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
 
 		private final Queue<PdbFile> pdbFiles;
 		private PdbReader reader;
+		private PdbFile currentPdbFile;
 
 		public EntrySupplier(final Collection<PdbFile> pdbFiles) {
 			super();
@@ -30,14 +31,15 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
 			if (reader == null) {
 				nextFile();
 			}
-			final Optional<Entry> optionalEntry = reader.readEntry();
+			final Optional<Entry> optionalEntry = reader.readEntry(currentPdbFile.getTags());
 
 			return optionalEntry.orElseGet(() -> {
 				nextFile();
 				if (reader == null) {
 					return null;
 				} else {
-					return reader.readEntry().orElse(null);
+					final Tags tags = currentPdbFile.getTags();
+					return reader.readEntry(tags).orElse(null);
 				}
 			});
 
@@ -51,16 +53,16 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
 			}
 
 			while (!pdbFiles.isEmpty()) {
-				final PdbFile pdbFile = pdbFiles.poll();
+				currentPdbFile = pdbFiles.poll();
 				try {
-					if (pdbFile.getFile().length() > 0) {
+					if (currentPdbFile.getFile().length() > 0) {
-						reader = new PdbReader(pdbFile);
+						reader = new PdbReader(currentPdbFile);
 						break;
 					} else {
-						LOGGER.info("ignoring empty file " + pdbFile);
+						LOGGER.info("ignoring empty file " + currentPdbFile);
 					}
 				} catch (final FileNotFoundException e) {
-					LOGGER.log(Level.WARNING, "the pdbFile " + pdbFile.getFile() + " is missing", e);
+					LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e);
 				}
 			}
 		}

performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java

@@ -0,0 +1,21 @@
+package org.lucares.performance.db;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+
+class PdbFileUtils {
+	static TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
+
+		try (PdbReader reader = new PdbReader(pdbFile)) {
+			if (reader.canSeekTail(2)) {
+				reader.seekTail(2);
+				final OffsetDateTime lastWrittenDate = reader.readDate();
+
+				return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo());
+			} else {
+				return pdbFile.getTimeRange();
+			}
+		}
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java

@@ -183,7 +183,7 @@ class PdbReader implements AutoCloseable {
 		}
 	}
 
-	public Optional<Entry> readEntry() throws ReadRuntimeException {
+	public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
 		final long epochMilli = readEpochMilli();
 		if (epochMilli < 0) {
 			return Optional.empty();
@@ -193,7 +193,7 @@ class PdbReader implements AutoCloseable {
 		if (value < 0) {
 			return Optional.empty();
 		}
-		return Optional.of(new Entry(epochMilli, value));
+		return Optional.of(new Entry(epochMilli, value, tags));
 	}
 
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java

@@ -1,21 +1,27 @@
 package org.lucares.performance.db;
 
 import java.io.BufferedOutputStream;
-import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 
 class PdbWriter implements AutoCloseable {
 
-	private static final long MAX_VALUE = 0xFF_FF_FF_FFL;
 	private static final boolean APPEND = true;
 	private final OutputStream outputStream;
 	private final PdbFile pdbFile;
+	private long minimalEpochMilli;
 
-	PdbWriter(final PdbFile pdbFile) throws FileNotFoundException {
+	PdbWriter(final PdbFile pdbFile) throws IOException {
 		this.pdbFile = pdbFile;
 		this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND));
+
+		if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) {
+			final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
+			minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli();
+		} else {
+			minimalEpochMilli = pdbFile.getTimeRange().getFrom().toInstant().toEpochMilli();
+		}
 	}
 
 	public PdbFile getFile() {
@@ -27,21 +33,29 @@ class PdbWriter implements AutoCloseable {
 	}
 
 	void write(final long epochMilli, final long value) throws WriteException {
-		final long offsetEpochMill = pdbFile.getOffsetInEpochMilli();
+		final long offsetEpochMilli = pdbFile.getOffsetInEpochMilli();
-		final long adjustedValue = epochMilli - offsetEpochMill;
+		final long adjustedValue = epochMilli - offsetEpochMilli;
 		assertValueInRange(adjustedValue);
 		assertValueInRange(value);
+		assertEpochMilliInRange(epochMilli);
 
 		write(adjustedValue);
 		write(value);
+		minimalEpochMilli = epochMilli;
+	}
+
+	private void assertEpochMilliInRange(final long epochMilli) {
+		if (epochMilli < minimalEpochMilli) {
+			throw new IllegalArgumentException("value must not be smaller than: " + minimalEpochMilli);
+		}
 	}
 
 	private void assertValueInRange(final long value) {
 		if (value < 0) {
 			throw new IllegalArgumentException("value must not be negative: " + value);
 		}
-		if (value > MAX_VALUE) {
+		if (value > Entry.MAX_VALUE) {
-			throw new IllegalArgumentException("max value is " + MAX_VALUE + " value was: " + value);
+			throw new IllegalArgumentException("max value is " + Entry.MAX_VALUE + " value was: " + value);
 		}
 	}
 

performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java

@@ -0,0 +1,93 @@
+package org.lucares.performance.db;
+
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class PdbWriterManager implements AutoCloseable {
+
+	private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName());
+
+	private final static class Key {
+		private final Tags tags;
+		private final Day day;
+
+		public Key(final Tags tags, final OffsetDateTime date) {
+			super();
+			this.tags = tags;
+			this.day = new Day(date);
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((day == null) ? 0 : day.hashCode());
+			result = prime * result + ((tags == null) ? 0 : tags.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj)
+				return true;
+			if (obj == null)
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			final Key other = (Key) obj;
+			if (day == null) {
+				if (other.day != null)
+					return false;
+			} else if (!day.equals(other.day))
+				return false;
+			if (tags == null) {
+				if (other.tags != null)
+					return false;
+			} else if (!tags.equals(other.tags))
+				return false;
+			return true;
+		}
+	}
+
+	public interface PdbWriterSupplier {
+		public PdbWriter supply(Tags tags, OffsetDateTime date);
+	}
+
+	final Map<Key, PdbWriter> map = new HashMap<>();
+
+	private final PdbWriterSupplier supplier;
+
+	public PdbWriterManager(final PdbWriterSupplier supplier) {
+		this.supplier = supplier;
+	}
+
+	public PdbWriter get(final Tags tags, final OffsetDateTime date) {
+
+		final Key key = new Key(tags, date);
+		if (!map.containsKey(key)) {
+			final PdbWriter writer = supplier.supply(tags, date);
+			put(tags, date, writer);
+		}
+		return map.get(key);
+	}
+
+	public PdbWriter put(final Tags tags, final OffsetDateTime date, final PdbWriter pdbWriter) {
+		final Key key = new Key(tags, date);
+		return map.put(key, pdbWriter);
+	}
+
+	@Override
+	public void close() {
+		for (final PdbWriter writer : map.values()) {
+			try {
+				writer.close();
+			} catch (final IOException e) {
+				LOGGER.log(Level.WARNING, e.getMessage(), e);
+			}
+		}
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -15,6 +15,8 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
+import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
+
 import liquibase.exception.LiquibaseException;
 
 public class PerformanceDb implements AutoCloseable {
@@ -26,38 +28,54 @@ public class PerformanceDb implements AutoCloseable {
 		tagsToFile = new TagsToFile(dataDirectory);
 	}
 
-	public void put(final OffsetDateTime date, final long value, final Tags tags) throws WriteException {
+	public void put(final Entry entry) throws WriteException {
-		put(new Entry(date, value), tags);
+		put(Arrays.asList(entry));
 	}
 
-	public void put(final Entry entry, final Tags tags) throws WriteException {
+	public void put(final Iterable<Entry> entries) throws WriteException {
-		put(Arrays.asList(entry), tags);
+		put(entries.iterator());
 	}
 
-	public void put(final Iterable<Entry> entries, final Tags tags) throws WriteException {
+	public void put(final BlockingQueue<Entry> entries, final Entry poisonObject) throws WriteException {
-		put(entries.iterator(), tags);
-	}
-
-	public void put(final BlockingQueue<Entry> entries, final Entry poisonObject, final Tags tags)
-			throws WriteException {
 		final BlockingQueueIterator<Entry> iterator = new BlockingQueueIterator<>(entries, poisonObject);
-		put(iterator, tags);
+		put(iterator);
 	}
 
-	public void put(final Iterator<Entry> entries, final Tags tags) throws WriteException {
+	public void put(final Iterator<Entry> entries) throws WriteException {
 
 		final BlockingIteratorIterator<Entry> iterator = new BlockingIteratorIterator<>(entries);
-		put(iterator, tags);
+		put(iterator);
 	}
 
-	public void put(final BlockingIterator<Entry> entries, final Tags tags) throws WriteException {
+	private static class WriterSupplier implements PdbWriterSupplier {
+
+		private final TagsToFile tagsToFile;
+
+		public WriterSupplier(final TagsToFile tagsToFile) {
+			super();
+			this.tagsToFile = tagsToFile;
+		}
+
+		@Override
+		public PdbWriter supply(final Tags tags, final OffsetDateTime date) {
+			try {
+				final PdbFile pdbFile = tagsToFile.getFile(date, tags);
+
+				final PdbWriter writer = new PdbWriter(pdbFile);
+				return writer;
+			} catch (final IOException e) {
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+	public void put(final BlockingIterator<Entry> entries) throws WriteException {
 
 		final long start = System.nanoTime();
 		final double timeSpendInWrite = 0.0;
 		long count = 0;
-		PdbWriter writer = null;
+
-		PdbFile pdbFile = null;
+		try (final PdbWriterManager manager = new PdbWriterManager(new WriterSupplier(tagsToFile));) {
-		try {
 			while (true) {
 				final Optional<Entry> entryOptional = entries.next();
 				if (!entryOptional.isPresent()) {
@@ -67,29 +85,10 @@ public class PerformanceDb implements AutoCloseable {
 
 				final long epochMilli = entry.getEpochMilli();
 				final long value = entry.getValue();
+				final Tags tags = entry.getTags();
+				final OffsetDateTime date = entry.getDate();
 
-				if (pdbFile == null //
+				final PdbWriter writer = manager.get(tags, date);
-						|| !pdbFile.getTimeRange().inRange(epochMilli)) // TODO
-																		// @ahr
-				// correct
-				// would be
-				// to check
-				// if the
-				// date is
-				// in the
-				// available
-				// range
-				{
-					final OffsetDateTime date = entry.getDate();
-					pdbFile = tagsToFile.getFile(date, tags);
-				}
-
-				if (writer == null || !writer.getFile().equals(pdbFile)) {
-					if (writer != null) {
-						writer.close();
-					}
-					writer = new PdbWriter(pdbFile);
-				}
 
 				writer.write(epochMilli, value);
 				count++;
@@ -98,17 +97,7 @@ public class PerformanceDb implements AutoCloseable {
 		} catch (final InterruptedException e) {
 			Thread.currentThread().interrupt();
 			LOGGER.info("Thread was interrupted. Aborting exectution.");
-		} catch (final IOException e) {
-			throw new WriteException(e);
 		} finally {
-			if (writer != null) {
-				try {
-					writer.close();
-				} catch (final IOException e) {
-					throw new WriteException(e);
-				}
-			}
-
 			final double duration = (System.nanoTime() - start) / 1_000_000.0;
 			LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite
 					+ " were spend in write");

performanceDb/src/main/java/org/lucares/performance/db/Tags.java

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
@@ -9,33 +10,55 @@ import java.util.TreeSet;
 import java.util.function.BiConsumer;
 
 public class Tags {
+
+	private static final Tags EMPTY = new Tags();
+
 	private final Map<String, String> tags = new HashMap<>();
 
-	public Tags() {
+	private Tags() {
 		super();
 	}
 
-	public Tags(final String key, final String value) {
+	private Tags(final Map<String, String> tags) {
-		super();
+		this.tags.putAll(tags);
-		tags.put(key, value);
 	}
 
-	public Tags(final String key1, final String value1, final String key2, final String value2) {
+	public static Tags create() {
-		super();
+		return EMPTY;
+	}
+
+	public static Tags create(final String key1, final String value1, final String key2, final String value2) {
+		final Map<String, String> tags = new HashMap<>(2);
 		tags.put(key1, value1);
 		tags.put(key2, value2);
+		return new Tags(tags); // TODO @ahr cache them
 	}
 
-	public Tags(final String key1, final String value1, final String key2, final String value2, final String key3,
+	public static Tags create(final String key1, final String value1) {
-			final String value3) {
+		final Map<String, String> tags = new HashMap<>(1);
-		super();
 		tags.put(key1, value1);
-		tags.put(key2, value2);
+		return new Tags(tags);
-		tags.put(key3, value3);
 	}
 
-	public void put(final String key, final String value) {
+	public Tags copyAdd(final String key, final String value) {
-		tags.put(key, value);
+		Objects.requireNonNull(key, "key must not be null");
+		Objects.requireNonNull(value, "value must not be null");
+
+		final Map<String, String> newTags = new HashMap<>(tags);
+		newTags.put(key.intern(), value.intern());
+
+		return new Tags(newTags);
+	}
+
+	public Tags copyAddIfNotNull(final String key, final String value) {
+
+		final Tags result;
+		if (value != null) {
+			result = copyAdd(key, value);
+		} else {
+			result = this;
+		}
+		return result;
 	}
 
 	public Optional<String> getValue(final String key) {
@@ -85,7 +108,7 @@ public class Tags {
 
 	public String abbreviatedRepresentation() {
 		final StringBuilder result = new StringBuilder();
-		final int maxLength = 200;
+		final int maxLength = 500;
 
 		final SortedSet<String> keys = new TreeSet<>(tags.keySet());
 
@@ -105,4 +128,5 @@ public class Tags {
 	private static String substr(final String s, final int maxLength) {
 		return s.substring(0, Math.min(maxLength, s.length()));
 	}
+
 }

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -63,7 +63,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 			// TODO @ahr ludb should handle unknown fields better
 		}
 
-		Collections.sort(result, new PdbFileByTimeAsc());
+		Collections.sort(result, PdbFileByTimeAsc.INSTANCE);
 		return result;
 	}
 
@@ -77,13 +77,13 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 	}
 
 	private Tags toTags(final Document document) {
-		final Tags tagsOfFile = new Tags();
+		Tags tagsOfFile = Tags.create();
 
 		for (final String key : document.getProperties().keySet()) {
 
 			if (Fields.isPrefixedKey(key)) {
 				final String value = document.getPropertyString(key);
-				tagsOfFile.put(Fields.stripPrefix(key), value);
+				tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value);
 			}
 		}
 		return tagsOfFile;
@@ -99,7 +99,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 			final boolean inRange = pdbFile.getTimeRange().inRange(date);
 
 			if (inRange) {
-				final TimeRange availableTimeRange = getAvailableTimeRange(pdbFile);
+				final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
 
 				if (availableTimeRange.inRange(date)) {
 					preResult.add(pdbFile);
@@ -119,20 +119,6 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
 		return result;
 	}
 
-	private TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
-
-		try (PdbReader reader = new PdbReader(pdbFile)) {
-			if (reader.canSeekTail(2)) {
-				reader.seekTail(2);
-				final OffsetDateTime lastWrittenDate = reader.readDate();
-
-				return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo());
-			} else {
-				return pdbFile.getTimeRange();
-			}
-		}
-	}
-
 	private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) {
 		final File file;
 		PdbFile result;

performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java

@@ -40,17 +40,18 @@ public class PdbReaderWriterTest {
 	public void testWriteRead(final long value) throws Exception {
 
 		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final PdbFile pdbFile = PdbFile.today(file, new Tags());
+		final Tags tags = Tags.create();
+		final PdbFile pdbFile = PdbFile.today(file, tags);
 		final OffsetDateTime now = OffsetDateTime.now(); // TODO @ahr might fail
 															// at midnight
-		final Entry entry = new Entry(now, value);
+		final Entry entry = new Entry(now, value, tags);
 
 		try (PdbWriter writer = new PdbWriter(pdbFile)) {
 			writer.write(entry);
 		}
 
 		try (final PdbReader reader = new PdbReader(pdbFile)) {
-			final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
+			final Entry actual = reader.readEntry(tags).orElseThrow(() -> new AssertionError());
 
 			Assert.assertEquals(actual, entry);
 		}
@@ -59,7 +60,7 @@ public class PdbReaderWriterTest {
 	public void testSeekTail() throws Exception {
 
 		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final PdbFile pdbFile = PdbFile.today(file, new Tags());
+		final PdbFile pdbFile = PdbFile.today(file, Tags.create());
 
 		try (PdbWriter writer = new PdbWriter(pdbFile)) {
 			writer.write(1);
@@ -86,7 +87,7 @@ public class PdbReaderWriterTest {
 	public void testSeek() throws Exception {
 
 		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final PdbFile pdbFile = PdbFile.today(file, new Tags());
+		final PdbFile pdbFile = PdbFile.today(file, Tags.create());
 
 		try (PdbWriter writer = new PdbWriter(pdbFile)) {
 			writer.write(1);

performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java

@@ -0,0 +1,59 @@
+package org.lucares.performance.db;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.OffsetDateTime;
+
+import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test
+public class PdbWriterManagerTest {
+
+	private Path dataDirectory;
+
+	@BeforeMethod
+	public void beforeMethod() throws IOException {
+		dataDirectory = Files.createTempDirectory("pdb");
+	}
+
+	@AfterMethod
+	public void afterMethod() throws IOException {
+		FileUtils.delete(dataDirectory);
+	}
+
+	@Test
+	public void testManager() throws Exception {
+
+		final PdbWriterSupplier supplier = (tags, date) -> {
+			File file;
+			try {
+				file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile();
+				return new PdbWriter(new PdbFile(new Day(date), file, tags));
+			} catch (final IOException e) {
+				throw new AssertionError(e.getMessage(), e);
+			}
+		};
+		final Tags tagsA = Tags.create("key", "A");
+		final Tags tagsB = Tags.create("key", "B");
+
+		try (PdbWriterManager manager = new PdbWriterManager(supplier)) {
+
+			final OffsetDateTime date = OffsetDateTime.now();
+
+			final PdbWriter firstWriterForTagsA = manager.get(tagsA, date);
+			final PdbWriter secondWriterForTagsA = manager.get(tagsA, date);
+			final PdbWriter firstWriterForTagsB = manager.get(tagsB, date);
+
+			Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
+			Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
+
+		}
+
+	}
+}

performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java

@@ -38,14 +38,14 @@ public class PerformanceDbTest {
 		try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
 			final OffsetDateTime date = DateUtils.nowInUtc();
 			final long value = 1;
-			final Tags tags = new Tags("myKey", "myValue");
+			final Tags tags = Tags.create("myKey", "myValue");
-			performanceDb.put(date, value, tags);
+			performanceDb.put(new Entry(date, value, tags));
 
 			final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
 
 			Assert.assertEquals(stream.size(), 1);
 
-			Assert.assertEquals(stream.get(0), new Entry(date, value));
+			Assert.assertEquals(stream.get(0), new Entry(date, value, tags));
 		}
 	}
 
@@ -56,28 +56,28 @@ public class PerformanceDbTest {
 			final OffsetDateTime dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56);
 			final long valueOne = 1;
 			final long valueTwo = 2;
-			final Tags tags = new Tags("myKey", "myValue");
+			final Tags tags = Tags.create("myKey", "myValue");
 
-			performanceDb.put(dayOne, valueOne, tags);
+			performanceDb.put(new Entry(dayOne, valueOne, tags));
-			performanceDb.put(dayTwo, valueTwo, tags);
+			performanceDb.put(new Entry(dayTwo, valueTwo, tags));
 
 			final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
 
 			Assert.assertEquals(stream.size(), 2);
 
-			Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne));
+			Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne, tags));
-			Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo));
+			Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo, tags));
 		}
 	}
 
-	private List<Entry> generateEntries(final TimeRange timeRange, final long n, final int addToDate) {
+	private List<Entry> generateEntries(final TimeRange timeRange, final long n, final int addToDate, final Tags tags) {
 		final List<Entry> result = new ArrayList<>();
 		final long differenceInMs = timeRange.duration().toMillis() / n;
 		long currentTime = timeRange.getFrom().toInstant().toEpochMilli();
 
 		for (long i = 0; i < n; i++) {
 			final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE);
-			result.add(new Entry(currentTime + addToDate, value));
+			result.add(new Entry(currentTime + addToDate, value, tags));
 
 			currentTime += differenceInMs;
 		}
@@ -91,13 +91,13 @@ public class PerformanceDbTest {
 			final TimeRange timeRange = TimeRange.ofDay(OffsetDateTime.now(ZoneOffset.UTC));
 			final long numberOfEntries = 2;
 
-			final Tags tags = new Tags("myKey", "one");
+			final Tags tags = Tags.create("myKey", "one");
-			final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0);
+			final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0, tags);
 
 			printEntries(entries, "");
 
 			for (final Entry entry : entries) {
-				performanceDb.put(entry, tags);
+				performanceDb.put(entry);
 			}
 
 			final List<Entry> actualEntries = performanceDb.getAsList(tags);
@@ -121,21 +121,21 @@ public class PerformanceDbTest {
 			final TimeRange timeRange = new TimeRange(from, to);
 			final long numberOfEntries = timeRange.duration().toHours();
 
-			final Tags tagsCommon = new Tags("commonKey", "commonValue");
+			final Tags tagsCommon = Tags.create("commonKey", "commonValue");
-			final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue");
+			final Tags tagsOne = Tags.create("myKey", "one", "commonKey", "commonValue");
-			final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries, 1);
+			final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries, 1, tagsOne);
 			printEntries(entriesOne, "one");
-			performanceDb.put(entriesOne, tagsOne);
+			performanceDb.put(entriesOne);
 
-			final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue");
+			final Tags tagsTwo = Tags.create("myKey", "two", "commonKey", "commonValue");
-			final List<Entry> entriesTwo = generateEntries(timeRange, numberOfEntries, 2);
+			final List<Entry> entriesTwo = generateEntries(timeRange, numberOfEntries, 2, tagsTwo);
 			printEntries(entriesTwo, "two");
-			performanceDb.put(entriesTwo, tagsTwo);
+			performanceDb.put(entriesTwo);
 
-			final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue");
+			final Tags tagsThree = Tags.create("myKey", "three", "commonKey", "commonValue");
-			final List<Entry> entriesThree = generateEntries(timeRange, numberOfEntries, 3);
+			final List<Entry> entriesThree = generateEntries(timeRange, numberOfEntries, 3, tagsThree);
 			printEntries(entriesThree, "three");
-			performanceDb.put(entriesThree, tagsThree);
+			performanceDb.put(entriesThree);
 
 			final List<Entry> actualEntriesOne = performanceDb.getAsList(tagsOne);
 			Assert.assertEquals(actualEntriesOne, entriesOne);

performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java

@@ -30,7 +30,7 @@ public class TagsToFilesTest {
 		try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) {
 
 			final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
-			final Tags tags = new Tags("myKey", "myValue");
+			final Tags tags = Tags.create("myKey", "myValue");
 
 			final PdbFile newFileForTags = tagsToFile.getFile(date, tags);
 			final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);

recommind-logs/src/main/java/org/lucares/recommind/logs/GrokToEntryFilter.java

@@ -0,0 +1,54 @@
+package org.lucares.recommind.logs;
+
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+
+import org.lucares.performance.db.DateUtils;
+import org.lucares.performance.db.Entry;
+import org.lucares.performance.db.Tags;
+
+import io.thekraken.grok.api.Grok;
+import io.thekraken.grok.api.Match;
+
+public class GrokToEntryFilter {
+
+	private final Grok grok;
+
+	private final static DateTimeFormatter FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS");
+
+	public GrokToEntryFilter(final Grok grok) {
+		this.grok = grok;
+	}
+
+	public Entry parse(final String singleLineOfLog, final Tags tags) {
+		final Match gm = grok.match(singleLineOfLog);
+		gm.captures();
+		final Map<String, Object> map = gm.toMap();
+
+		final String timestamp = (String) map.get("timestamp");
+		final String method = (String) map.get("method");
+		final String project = (String) map.get("project");
+		final Long value = (Long) map.get("duration");
+
+		final Entry result;
+		if (timestamp == null || method == null || value == null) {
+			result = null;
+		} else {
+
+			final OffsetDateTime date = DateUtils.parseAtZoneOffset(timestamp, FORMAT, ZoneOffset.UTC);
+
+			Tags entryTags = tags;
+			entryTags = entryTags.copyAddIfNotNull("method", method);
+			entryTags = entryTags.copyAddIfNotNull("project", lowerCase(project));
+
+			result = new Entry(date, value, entryTags);
+		}
+		return result;
+	}
+
+	private String lowerCase(final String s) {
+		return s != null ? s.toLowerCase().intern() : s;
+	}
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/Ingestor.java

@@ -0,0 +1,74 @@
+package org.lucares.recommind.logs;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+import org.lucares.performance.db.PerformanceDb;
+import org.lucares.performance.db.Tags;
+
+import io.thekraken.grok.api.Grok;
+import io.thekraken.grok.api.exception.GrokException;
+import liquibase.exception.LiquibaseException;
+
+public class Ingestor {
+
+	public static String createPattern() {
+		// "%{TIMESTAMP_ISO8601:timestamp}\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?<logger>(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null
+		// - Executed
+		// %{NOTSPACE:method} in %{NUMBER:duration} ms %{WORD:status}.
+		// \\[.*?(?:project=%{WORD:project})?"
+		final String time = "%{TIMESTAMP_ISO8601:timestamp}";
+		final String stuff = "\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?<logger>(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null - Executed ";
+		final String method = "%{NOTSPACE:method}";
+		final String duration = "%{NUMBER:duration:long}";
+		final String status = "%{WORD:status}";
+		final String project = "(?:project=%{WORD:project})?";
+		return time + stuff + method + " in " + duration + " ms " + status + ". \\[.*?" + project;
+		// return time;
+
+	}
+
+	public static void main(final String[] args) throws LiquibaseException, Exception {
+		final Path dataDirectory = Files.createTempDirectory("ingestor");
+		final File logFile = new File(
+				"/home/andi/ws/performanceDb/data/production/ondem/ondem01/ap001/1_performance.log");
+
+		final Grok grok = createGrok(dataDirectory);
+
+		grok.compile(createPattern());
+
+		try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+
+			final PerformanceLogs performanceLogs = new PerformanceLogs();
+			final Tags tags = Tags.create("pod", "ondem01");
+
+			final long start = System.nanoTime();
+
+			performanceLogs.ingest(db, logFile, tags, grok);
+
+			System.out.println("duration: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
+		}
+	}
+
+	private static Grok createGrok(final Path dataDirectory) throws GrokException, IOException {
+
+		final File patternsFile = storePatterns(dataDirectory);
+
+		return Grok.create(patternsFile.getAbsolutePath());
+	}
+
+	private static File storePatterns(final Path dataDirectory) throws IOException {
+
+		final File file = new File(dataDirectory.toFile(), "patterns");
+		try (InputStream stream = Ingestor.class.getClassLoader().getResourceAsStream("org/lucares/grok/patterns")) {
+
+			Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+		}
+
+		return file;
+	}
+}

recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java

@@ -1,19 +1,70 @@
 package org.lucares.recommind.logs;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
 import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
+import org.lucares.performance.db.BlockingIterator;
+import org.lucares.performance.db.BlockingQueueIterator;
 import org.lucares.performance.db.Entry;
 import org.lucares.performance.db.PerformanceDb;
 import org.lucares.performance.db.Tags;
 
+import io.thekraken.grok.api.Grok;
+
 public class PerformanceLogs {
 
-	public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags) {
+	private final ExecutorService executor = Executors.newCachedThreadPool();
+
+	public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags, final Grok grok)
+			throws InterruptedException {
 
 		final ArrayBlockingQueue<Entry> queue = new ArrayBlockingQueue<>(10);
 
-		db.put(queue, tags);
+		final BlockingIterator<Entry> iterator = new BlockingQueueIterator<>(queue, Entry.POISON);
 
+		final Future<Boolean> future = executor.submit(() -> {
+
+			final GrokToEntryFilter filter = new GrokToEntryFilter(grok);
+
+			boolean result = false;
+
+			try (final BufferedReader reader = new BufferedReader(new FileReader(performanceLog))) {
+				String line;
+				while ((line = reader.readLine()) != null) {
+					final Entry entry = filter.parse(line, tags);
+					if (entry != null) {
+						System.out.println(entry);
+						queue.put(entry);
+					}
+				}
+
+				result = true;
+			} finally {
+				queue.put(Entry.POISON);
+			}
+			return result;
+		});
+
+		try {
+			db.put(iterator);
+			try {
+				future.get(10, TimeUnit.MINUTES);
+			} catch (ExecutionException | TimeoutException e) {
+				e.printStackTrace(); // TODO @ahr handle this mess
+			}
+		} catch (final RuntimeException e) {
+			future.cancel(true);
+		} finally {
+			executor.shutdown();
+			executor.awaitTermination(10, TimeUnit.MINUTES);
+		}
 	}
 }

recommind-logs/src/main/resources/org/lucares/grok/patterns

@@ -0,0 +1,94 @@
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+IP (?:%{IPV6}|%{IPV4})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME}
+IPORHOST (?:%{HOSTNAME}|%{IP})
+HOSTPORT %{IPORHOST}:%{POSINT}
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+
+# Log Levels
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)