From 34ee64fff1eb38cfa059da1370f916d692349195 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 10 Dec 2016 14:10:41 +0100 Subject: [PATCH] insert entries for different tags in one stream --- .../performance/db/BlockingQueueIterator.java | 2 +- .../org/lucares/performance/db/DateUtils.java | 29 ++++-- .../org/lucares/performance/db/Entry.java | 41 ++++++-- .../performance/db/PdbFileByTimeAsc.java | 2 + .../performance/db/PdbFileIterator.java | 16 ++-- .../lucares/performance/db/PdbFileUtils.java | 21 +++++ .../org/lucares/performance/db/PdbReader.java | 4 +- .../org/lucares/performance/db/PdbWriter.java | 28 ++++-- .../performance/db/PdbWriterManager.java | 93 ++++++++++++++++++ .../lucares/performance/db/PerformanceDb.java | 87 ++++++++--------- .../java/org/lucares/performance/db/Tags.java | 52 +++++++--- .../lucares/performance/db/TagsToFile.java | 22 +---- .../performance/db/PdbReaderWriterTest.java | 11 ++- .../performance/db/PdbWriterManagerTest.java | 59 ++++++++++++ .../performance/db/PerformanceDbTest.java | 46 ++++----- .../performance/db/TagsToFilesTest.java | 2 +- .../recommind/logs/GrokToEntryFilter.java | 54 +++++++++++ .../org/lucares/recommind/logs/Ingestor.java | 74 +++++++++++++++ .../recommind/logs/PerformanceLogs.java | 55 ++++++++++- .../main/resources/org/lucares/grok/patterns | 94 +++++++++++++++++++ 20 files changed, 648 insertions(+), 144 deletions(-) create mode 100644 performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java create mode 100644 performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java create mode 100644 performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/GrokToEntryFilter.java create mode 100644 recommind-logs/src/main/java/org/lucares/recommind/logs/Ingestor.java create mode 100644 recommind-logs/src/main/resources/org/lucares/grok/patterns diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java index c882d9f..d20c337 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java @@ -3,7 +3,7 @@ package org.lucares.performance.db; import java.util.Optional; import java.util.concurrent.BlockingQueue; -final class BlockingQueueIterator implements BlockingIterator { +public final class BlockingQueueIterator implements BlockingIterator { private final BlockingQueue queue; diff --git a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java index ebfe249..d829269 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java @@ -1,11 +1,14 @@ package org.lucares.performance.db; -import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalTime; import java.time.OffsetDateTime; import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQuery; import java.util.Calendar; -import java.util.Date; import java.util.TimeZone; public class DateUtils { @@ -33,14 +36,22 @@ public class DateUtils { return result; } - public static String format(final Date date) { - final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH-mm:ss,SSS Z"); - dateFormat.setTimeZone(TIME_ZONE_UTC); - - return dateFormat.format(date); - } - public static OffsetDateTime nowInUtc() { return OffsetDateTime.now(ZoneOffset.UTC); } + + public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter, + final ZoneOffset zoneOffset) { + final TemporalQuery query = new TemporalQuery() { + + @Override + public OffsetDateTime queryFrom(final TemporalAccessor temporal) { + final LocalDate localDate = LocalDate.from(temporal); + final LocalTime localTime = LocalTime.from(temporal); + return OffsetDateTime.of(localDate, localTime, zoneOffset); + } + }; + final OffsetDateTime date = formatter.parse(text, query); + return date; + } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Entry.java b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java index dd7e611..2d54cef 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Entry.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java @@ -9,20 +9,40 @@ import java.util.Comparator; public class Entry { public static final Comparator BY_DATE = new EntryByDateComparator(); + /** + * A special {@link Entry} that can be used as poison object for + * {@link BlockingQueueIterator}. + */ + public static final Entry POISON = new Entry(0, -1); + + public static final long MAX_VALUE = 0xFF_FF_FF_FFL; + private final long epochMilli; private final long value; - public Entry(final OffsetDateTime date, final long value) { - super(); + private final Tags tags; + + public Entry(final OffsetDateTime date, final long value, final Tags tags) { + this.tags = tags; this.epochMilli = date.toInstant().toEpochMilli(); this.value = value; } - Entry(final long epochMilli, final long value) { - super(); + Entry(final long epochMilli, final long value, final Tags tags) { + if (value < 0 || value > MAX_VALUE) { + throw new IllegalArgumentException("value must be between 0 and " + MAX_VALUE); + } + this.epochMilli = epochMilli; this.value = value; + this.tags = tags; + } + + private Entry(final long epochMilli, final long value) { + this.epochMilli = epochMilli; + this.value = value; + this.tags = null; } public OffsetDateTime getDate() { @@ -38,10 +58,14 @@ public class Entry { return epochMilli; } + public Tags getTags() { + return tags; + } + @Override public String toString() { final OffsetDateTime date = getDate(); - return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value; + return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags + ")"; } @Override @@ -49,6 +73,7 @@ public class Entry { final int prime = 31; int result = 1; result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32)); + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); result = prime * result + (int) (value ^ (value >>> 32)); return result; } @@ -64,9 +89,13 @@ public class Entry { final Entry other = (Entry) obj; if (epochMilli != other.epochMilli) return false; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; if (value != other.value) return false; return true; } - } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java index ac66b0f..7a1c4ac 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java @@ -5,6 +5,8 @@ import java.util.Comparator; public class PdbFileByTimeAsc implements Comparator { + public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc(); + @Override public int compare(final PdbFile o1, final PdbFile o2) { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index ea9e917..83135da 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -18,6 +18,7 @@ public class PdbFileIterator implements Iterator, AutoCloseable { private final Queue pdbFiles; private PdbReader reader; + private PdbFile currentPdbFile; public EntrySupplier(final Collection pdbFiles) { super(); @@ -30,14 +31,15 @@ public class PdbFileIterator implements Iterator, AutoCloseable { if (reader == null) { nextFile(); } - final Optional optionalEntry = reader.readEntry(); + final Optional optionalEntry = reader.readEntry(currentPdbFile.getTags()); return optionalEntry.orElseGet(() -> { nextFile(); if (reader == null) { return null; } else { - return reader.readEntry().orElse(null); + final Tags tags = currentPdbFile.getTags(); + return reader.readEntry(tags).orElse(null); } }); @@ -51,16 +53,16 @@ public class PdbFileIterator implements Iterator, AutoCloseable { } while (!pdbFiles.isEmpty()) { - final PdbFile pdbFile = pdbFiles.poll(); + currentPdbFile = pdbFiles.poll(); try { - if (pdbFile.getFile().length() > 0) { - reader = new PdbReader(pdbFile); + if (currentPdbFile.getFile().length() > 0) { + reader = new PdbReader(currentPdbFile); break; } else { - LOGGER.info("ignoring empty file " + pdbFile); + LOGGER.info("ignoring empty file " + currentPdbFile); } } catch (final FileNotFoundException e) { - LOGGER.log(Level.WARNING, "the pdbFile " + pdbFile.getFile() + " is missing", e); + LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e); } } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java new file mode 100644 index 0000000..2385fbe --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java @@ -0,0 +1,21 @@ +package org.lucares.performance.db; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.time.OffsetDateTime; + +class PdbFileUtils { + static TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException { + + try (PdbReader reader = new PdbReader(pdbFile)) { + if (reader.canSeekTail(2)) { + reader.seekTail(2); + final OffsetDateTime lastWrittenDate = reader.readDate(); + + return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo()); + } else { + return pdbFile.getTimeRange(); + } + } + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index 82fc093..711c5c4 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -183,7 +183,7 @@ class PdbReader implements AutoCloseable { } } - public Optional readEntry() throws ReadRuntimeException { + public Optional readEntry(final Tags tags) throws ReadRuntimeException { final long epochMilli = readEpochMilli(); if (epochMilli < 0) { return Optional.empty(); @@ -193,7 +193,7 @@ class PdbReader implements AutoCloseable { if (value < 0) { return Optional.empty(); } - return Optional.of(new Entry(epochMilli, value)); + return Optional.of(new Entry(epochMilli, value, tags)); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java index d292bc1..49e2c41 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -1,21 +1,27 @@ package org.lucares.performance.db; import java.io.BufferedOutputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; class PdbWriter implements AutoCloseable { - private static final long MAX_VALUE = 0xFF_FF_FF_FFL; private static final boolean APPEND = true; private final OutputStream outputStream; private final PdbFile pdbFile; + private long minimalEpochMilli; - PdbWriter(final PdbFile pdbFile) throws FileNotFoundException { + PdbWriter(final PdbFile pdbFile) throws IOException { this.pdbFile = pdbFile; this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND)); + + if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) { + final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile); + minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli(); + } else { + minimalEpochMilli = pdbFile.getTimeRange().getFrom().toInstant().toEpochMilli(); + } } public PdbFile getFile() { @@ -27,21 +33,29 @@ class PdbWriter implements AutoCloseable { } void write(final long epochMilli, final long value) throws WriteException { - final long offsetEpochMill = pdbFile.getOffsetInEpochMilli(); - final long adjustedValue = epochMilli - offsetEpochMill; + final long offsetEpochMilli = pdbFile.getOffsetInEpochMilli(); + final long adjustedValue = epochMilli - offsetEpochMilli; assertValueInRange(adjustedValue); assertValueInRange(value); + assertEpochMilliInRange(epochMilli); write(adjustedValue); write(value); + minimalEpochMilli = epochMilli; + } + + private void assertEpochMilliInRange(final long epochMilli) { + if (epochMilli < minimalEpochMilli) { + throw new IllegalArgumentException("value must not be smaller than: " + minimalEpochMilli); + } } private void assertValueInRange(final long value) { if (value < 0) { throw new IllegalArgumentException("value must not be negative: " + value); } - if (value > MAX_VALUE) { - throw new IllegalArgumentException("max value is " + MAX_VALUE + " value was: " + value); + if (value > Entry.MAX_VALUE) { + throw new IllegalArgumentException("max value is " + Entry.MAX_VALUE + " value was: " + value); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java new file mode 100644 index 0000000..54e8572 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java @@ -0,0 +1,93 @@ +package org.lucares.performance.db; + +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class PdbWriterManager implements AutoCloseable { + + private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName()); + + private final static class Key { + private final Tags tags; + private final Day day; + + public Key(final Tags tags, final OffsetDateTime date) { + super(); + this.tags = tags; + this.day = new Day(date); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((day == null) ? 0 : day.hashCode()); + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Key other = (Key) obj; + if (day == null) { + if (other.day != null) + return false; + } else if (!day.equals(other.day)) + return false; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; + return true; + } + } + + public interface PdbWriterSupplier { + public PdbWriter supply(Tags tags, OffsetDateTime date); + } + + final Map map = new HashMap<>(); + + private final PdbWriterSupplier supplier; + + public PdbWriterManager(final PdbWriterSupplier supplier) { + this.supplier = supplier; + } + + public PdbWriter get(final Tags tags, final OffsetDateTime date) { + + final Key key = new Key(tags, date); + if (!map.containsKey(key)) { + final PdbWriter writer = supplier.supply(tags, date); + put(tags, date, writer); + } + return map.get(key); + } + + public PdbWriter put(final Tags tags, final OffsetDateTime date, final PdbWriter pdbWriter) { + final Key key = new Key(tags, date); + return map.put(key, pdbWriter); + } + + @Override + public void close() { + for (final PdbWriter writer : map.values()) { + try { + writer.close(); + } catch (final IOException e) { + LOGGER.log(Level.WARNING, e.getMessage(), e); + } + } + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index f284ef6..0e47e61 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -15,6 +15,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier; + import liquibase.exception.LiquibaseException; public class PerformanceDb implements AutoCloseable { @@ -26,38 +28,54 @@ public class PerformanceDb implements AutoCloseable { tagsToFile = new TagsToFile(dataDirectory); } - public void put(final OffsetDateTime date, final long value, final Tags tags) throws WriteException { - put(new Entry(date, value), tags); + public void put(final Entry entry) throws WriteException { + put(Arrays.asList(entry)); } - public void put(final Entry entry, final Tags tags) throws WriteException { - put(Arrays.asList(entry), tags); + public void put(final Iterable entries) throws WriteException { + put(entries.iterator()); } - public void put(final Iterable entries, final Tags tags) throws WriteException { - put(entries.iterator(), tags); - } - - public void put(final BlockingQueue entries, final Entry poisonObject, final Tags tags) - throws WriteException { + public void put(final BlockingQueue entries, final Entry poisonObject) throws WriteException { final BlockingQueueIterator iterator = new BlockingQueueIterator<>(entries, poisonObject); - put(iterator, tags); + put(iterator); } - public void put(final Iterator entries, final Tags tags) throws WriteException { + public void put(final Iterator entries) throws WriteException { final BlockingIteratorIterator iterator = new BlockingIteratorIterator<>(entries); - put(iterator, tags); + put(iterator); } - public void put(final BlockingIterator entries, final Tags tags) throws WriteException { + private static class WriterSupplier implements PdbWriterSupplier { + + private final TagsToFile tagsToFile; + + public WriterSupplier(final TagsToFile tagsToFile) { + super(); + this.tagsToFile = tagsToFile; + } + + @Override + public PdbWriter supply(final Tags tags, final OffsetDateTime date) { + try { + final PdbFile pdbFile = tagsToFile.getFile(date, tags); + + final PdbWriter writer = new PdbWriter(pdbFile); + return writer; + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + } + + public void put(final BlockingIterator entries) throws WriteException { final long start = System.nanoTime(); final double timeSpendInWrite = 0.0; long count = 0; - PdbWriter writer = null; - PdbFile pdbFile = null; - try { + + try (final PdbWriterManager manager = new PdbWriterManager(new WriterSupplier(tagsToFile));) { while (true) { final Optional entryOptional = entries.next(); if (!entryOptional.isPresent()) { @@ -67,29 +85,10 @@ public class PerformanceDb implements AutoCloseable { final long epochMilli = entry.getEpochMilli(); final long value = entry.getValue(); + final Tags tags = entry.getTags(); + final OffsetDateTime date = entry.getDate(); - if (pdbFile == null // - || !pdbFile.getTimeRange().inRange(epochMilli)) // TODO - // @ahr - // correct - // would be - // to check - // if the - // date is - // in the - // available - // range - { - final OffsetDateTime date = entry.getDate(); - pdbFile = tagsToFile.getFile(date, tags); - } - - if (writer == null || !writer.getFile().equals(pdbFile)) { - if (writer != null) { - writer.close(); - } - writer = new PdbWriter(pdbFile); - } + final PdbWriter writer = manager.get(tags, date); writer.write(epochMilli, value); count++; @@ -98,17 +97,7 @@ public class PerformanceDb implements AutoCloseable { } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.info("Thread was interrupted. Aborting exectution."); - } catch (final IOException e) { - throw new WriteException(e); } finally { - if (writer != null) { - try { - writer.close(); - } catch (final IOException e) { - throw new WriteException(e); - } - } - final double duration = (System.nanoTime() - start) / 1_000_000.0; LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite + " were spend in write"); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Tags.java b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java index 97a5528..8afa573 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Tags.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java @@ -2,6 +2,7 @@ package org.lucares.performance.db; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedSet; @@ -9,33 +10,55 @@ import java.util.TreeSet; import java.util.function.BiConsumer; public class Tags { + + private static final Tags EMPTY = new Tags(); + private final Map tags = new HashMap<>(); - public Tags() { + private Tags() { super(); } - public Tags(final String key, final String value) { - super(); - tags.put(key, value); + private Tags(final Map tags) { + this.tags.putAll(tags); } - public Tags(final String key1, final String value1, final String key2, final String value2) { - super(); + public static Tags create() { + return EMPTY; + } + + public static Tags create(final String key1, final String value1, final String key2, final String value2) { + final Map tags = new HashMap<>(2); tags.put(key1, value1); tags.put(key2, value2); + return new Tags(tags); // TODO @ahr cache them } - public Tags(final String key1, final String value1, final String key2, final String value2, final String key3, - final String value3) { - super(); + public static Tags create(final String key1, final String value1) { + final Map tags = new HashMap<>(1); tags.put(key1, value1); - tags.put(key2, value2); - tags.put(key3, value3); + return new Tags(tags); } - public void put(final String key, final String value) { - tags.put(key, value); + public Tags copyAdd(final String key, final String value) { + Objects.requireNonNull(key, "key must not be null"); + Objects.requireNonNull(value, "value must not be null"); + + final Map newTags = new HashMap<>(tags); + newTags.put(key.intern(), value.intern()); + + return new Tags(newTags); + } + + public Tags copyAddIfNotNull(final String key, final String value) { + + final Tags result; + if (value != null) { + result = copyAdd(key, value); + } else { + result = this; + } + return result; } public Optional getValue(final String key) { @@ -85,7 +108,7 @@ public class Tags { public String abbreviatedRepresentation() { final StringBuilder result = new StringBuilder(); - final int maxLength = 200; + final int maxLength = 500; final SortedSet keys = new TreeSet<>(tags.keySet()); @@ -105,4 +128,5 @@ public class Tags { private static String substr(final String s, final int maxLength) { return s.substring(0, Math.min(maxLength, s.length())); } + } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index b1a151e..db5694d 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -63,7 +63,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { // TODO @ahr ludb should handle unknown fields better } - Collections.sort(result, new PdbFileByTimeAsc()); + Collections.sort(result, PdbFileByTimeAsc.INSTANCE); return result; } @@ -77,13 +77,13 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { } private Tags toTags(final Document document) { - final Tags tagsOfFile = new Tags(); + Tags tagsOfFile = Tags.create(); for (final String key : document.getProperties().keySet()) { if (Fields.isPrefixedKey(key)) { final String value = document.getPropertyString(key); - tagsOfFile.put(Fields.stripPrefix(key), value); + tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value); } } return tagsOfFile; @@ -99,7 +99,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { final boolean inRange = pdbFile.getTimeRange().inRange(date); if (inRange) { - final TimeRange availableTimeRange = getAvailableTimeRange(pdbFile); + final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile); if (availableTimeRange.inRange(date)) { preResult.add(pdbFile); @@ -119,20 +119,6 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { return result; } - private TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException { - - try (PdbReader reader = new PdbReader(pdbFile)) { - if (reader.canSeekTail(2)) { - reader.seekTail(2); - final OffsetDateTime lastWrittenDate = reader.readDate(); - - return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo()); - } else { - return pdbFile.getTimeRange(); - } - } - } - private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) { final File file; PdbFile result; diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java index dcc80ab..093fbf7 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java @@ -40,17 +40,18 @@ public class PdbReaderWriterTest { public void testWriteRead(final long value) throws Exception { final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); - final PdbFile pdbFile = PdbFile.today(file, new Tags()); + final Tags tags = Tags.create(); + final PdbFile pdbFile = PdbFile.today(file, tags); final OffsetDateTime now = OffsetDateTime.now(); // TODO @ahr might fail // at midnight - final Entry entry = new Entry(now, value); + final Entry entry = new Entry(now, value, tags); try (PdbWriter writer = new PdbWriter(pdbFile)) { writer.write(entry); } try (final PdbReader reader = new PdbReader(pdbFile)) { - final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError()); + final Entry actual = reader.readEntry(tags).orElseThrow(() -> new AssertionError()); Assert.assertEquals(actual, entry); } @@ -59,7 +60,7 @@ public class PdbReaderWriterTest { public void testSeekTail() throws Exception { final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); - final PdbFile pdbFile = PdbFile.today(file, new Tags()); + final PdbFile pdbFile = PdbFile.today(file, Tags.create()); try (PdbWriter writer = new PdbWriter(pdbFile)) { writer.write(1); @@ -86,7 +87,7 @@ public class PdbReaderWriterTest { public void testSeek() throws Exception { final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); - final PdbFile pdbFile = PdbFile.today(file, new Tags()); + final PdbFile pdbFile = PdbFile.today(file, Tags.create()); try (PdbWriter writer = new PdbWriter(pdbFile)) { writer.write(1); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java new file mode 100644 index 0000000..a7d1cd7 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java @@ -0,0 +1,59 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.OffsetDateTime; + +import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test +public class PdbWriterManagerTest { + + private Path dataDirectory; + + @BeforeMethod + public void beforeMethod() throws IOException { + dataDirectory = Files.createTempDirectory("pdb"); + } + + @AfterMethod + public void afterMethod() throws IOException { + FileUtils.delete(dataDirectory); + } + + @Test + public void testManager() throws Exception { + + final PdbWriterSupplier supplier = (tags, date) -> { + File file; + try { + file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile(); + return new PdbWriter(new PdbFile(new Day(date), file, tags)); + } catch (final IOException e) { + throw new AssertionError(e.getMessage(), e); + } + }; + final Tags tagsA = Tags.create("key", "A"); + final Tags tagsB = Tags.create("key", "B"); + + try (PdbWriterManager manager = new PdbWriterManager(supplier)) { + + final OffsetDateTime date = OffsetDateTime.now(); + + final PdbWriter firstWriterForTagsA = manager.get(tagsA, date); + final PdbWriter secondWriterForTagsA = manager.get(tagsA, date); + final PdbWriter firstWriterForTagsB = manager.get(tagsB, date); + + Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA); + Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB); + + } + + } +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java index 70759ae..f69b96f 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -38,14 +38,14 @@ public class PerformanceDbTest { try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { final OffsetDateTime date = DateUtils.nowInUtc(); final long value = 1; - final Tags tags = new Tags("myKey", "myValue"); - performanceDb.put(date, value, tags); + final Tags tags = Tags.create("myKey", "myValue"); + performanceDb.put(new Entry(date, value, tags)); final List stream = performanceDb.get(tags).collect(Collectors.toList()); Assert.assertEquals(stream.size(), 1); - Assert.assertEquals(stream.get(0), new Entry(date, value)); + Assert.assertEquals(stream.get(0), new Entry(date, value, tags)); } } @@ -56,28 +56,28 @@ public class PerformanceDbTest { final OffsetDateTime dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56); final long valueOne = 1; final long valueTwo = 2; - final Tags tags = new Tags("myKey", "myValue"); + final Tags tags = Tags.create("myKey", "myValue"); - performanceDb.put(dayOne, valueOne, tags); - performanceDb.put(dayTwo, valueTwo, tags); + performanceDb.put(new Entry(dayOne, valueOne, tags)); + performanceDb.put(new Entry(dayTwo, valueTwo, tags)); final List stream = performanceDb.get(tags).collect(Collectors.toList()); Assert.assertEquals(stream.size(), 2); - Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne)); - Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo)); + Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne, tags)); + Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo, tags)); } } - private List generateEntries(final TimeRange timeRange, final long n, final int addToDate) { + private List generateEntries(final TimeRange timeRange, final long n, final int addToDate, final Tags tags) { final List result = new ArrayList<>(); final long differenceInMs = timeRange.duration().toMillis() / n; long currentTime = timeRange.getFrom().toInstant().toEpochMilli(); for (long i = 0; i < n; i++) { final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE); - result.add(new Entry(currentTime + addToDate, value)); + result.add(new Entry(currentTime + addToDate, value, tags)); currentTime += differenceInMs; } @@ -91,13 +91,13 @@ public class PerformanceDbTest { final TimeRange timeRange = TimeRange.ofDay(OffsetDateTime.now(ZoneOffset.UTC)); final long numberOfEntries = 2; - final Tags tags = new Tags("myKey", "one"); - final List entries = generateEntries(timeRange, numberOfEntries, 0); + final Tags tags = Tags.create("myKey", "one"); + final List entries = generateEntries(timeRange, numberOfEntries, 0, tags); printEntries(entries, ""); for (final Entry entry : entries) { - performanceDb.put(entry, tags); + performanceDb.put(entry); } final List actualEntries = performanceDb.getAsList(tags); @@ -121,21 +121,21 @@ public class PerformanceDbTest { final TimeRange timeRange = new TimeRange(from, to); final long numberOfEntries = timeRange.duration().toHours(); - final Tags tagsCommon = new Tags("commonKey", "commonValue"); - final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue"); - final List entriesOne = generateEntries(timeRange, numberOfEntries, 1); + final Tags tagsCommon = Tags.create("commonKey", "commonValue"); + final Tags tagsOne = Tags.create("myKey", "one", "commonKey", "commonValue"); + final List entriesOne = generateEntries(timeRange, numberOfEntries, 1, tagsOne); printEntries(entriesOne, "one"); - performanceDb.put(entriesOne, tagsOne); + performanceDb.put(entriesOne); - final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue"); - final List entriesTwo = generateEntries(timeRange, numberOfEntries, 2); + final Tags tagsTwo = Tags.create("myKey", "two", "commonKey", "commonValue"); + final List entriesTwo = generateEntries(timeRange, numberOfEntries, 2, tagsTwo); printEntries(entriesTwo, "two"); - performanceDb.put(entriesTwo, tagsTwo); + performanceDb.put(entriesTwo); - final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue"); - final List entriesThree = generateEntries(timeRange, numberOfEntries, 3); + final Tags tagsThree = Tags.create("myKey", "three", "commonKey", "commonValue"); + final List entriesThree = generateEntries(timeRange, numberOfEntries, 3, tagsThree); printEntries(entriesThree, "three"); - performanceDb.put(entriesThree, tagsThree); + performanceDb.put(entriesThree); final List actualEntriesOne = performanceDb.getAsList(tagsOne); Assert.assertEquals(actualEntriesOne, entriesOne); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java index a991668..4c87f88 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -30,7 +30,7 @@ public class TagsToFilesTest { try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) { final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); - final Tags tags = new Tags("myKey", "myValue"); + final Tags tags = Tags.create("myKey", "myValue"); final PdbFile newFileForTags = tagsToFile.getFile(date, tags); final PdbFile existingFileForTags = tagsToFile.getFile(date, tags); diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/GrokToEntryFilter.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/GrokToEntryFilter.java new file mode 100644 index 0000000..f21d257 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/GrokToEntryFilter.java @@ -0,0 +1,54 @@ +package org.lucares.recommind.logs; + +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Map; + +import org.lucares.performance.db.DateUtils; +import org.lucares.performance.db.Entry; +import org.lucares.performance.db.Tags; + +import io.thekraken.grok.api.Grok; +import io.thekraken.grok.api.Match; + +public class GrokToEntryFilter { + + private final Grok grok; + + private final static DateTimeFormatter FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); + + public GrokToEntryFilter(final Grok grok) { + this.grok = grok; + } + + public Entry parse(final String singleLineOfLog, final Tags tags) { + final Match gm = grok.match(singleLineOfLog); + gm.captures(); + final Map map = gm.toMap(); + + final String timestamp = (String) map.get("timestamp"); + final String method = (String) map.get("method"); + final String project = (String) map.get("project"); + final Long value = (Long) map.get("duration"); + + final Entry result; + if (timestamp == null || method == null || value == null) { + result = null; + } else { + + final OffsetDateTime date = DateUtils.parseAtZoneOffset(timestamp, FORMAT, ZoneOffset.UTC); + + Tags entryTags = tags; + entryTags = entryTags.copyAddIfNotNull("method", method); + entryTags = entryTags.copyAddIfNotNull("project", lowerCase(project)); + + result = new Entry(date, value, entryTags); + } + return result; + } + + private String lowerCase(final String s) { + return s != null ? s.toLowerCase().intern() : s; + } +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/Ingestor.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/Ingestor.java new file mode 100644 index 0000000..f606ac2 --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/Ingestor.java @@ -0,0 +1,74 @@ +package org.lucares.recommind.logs; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.lucares.performance.db.PerformanceDb; +import org.lucares.performance.db.Tags; + +import io.thekraken.grok.api.Grok; +import io.thekraken.grok.api.exception.GrokException; +import liquibase.exception.LiquibaseException; + +public class Ingestor { + + public static String createPattern() { + // "%{TIMESTAMP_ISO8601:timestamp}\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null + // - Executed + // %{NOTSPACE:method} in %{NUMBER:duration} ms %{WORD:status}. + // \\[.*?(?:project=%{WORD:project})?" + final String time = "%{TIMESTAMP_ISO8601:timestamp}"; + final String stuff = "\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null - Executed "; + final String method = "%{NOTSPACE:method}"; + final String duration = "%{NUMBER:duration:long}"; + final String status = "%{WORD:status}"; + final String project = "(?:project=%{WORD:project})?"; + return time + stuff + method + " in " + duration + " ms " + status + ". \\[.*?" + project; + // return time; + + } + + public static void main(final String[] args) throws LiquibaseException, Exception { + final Path dataDirectory = Files.createTempDirectory("ingestor"); + final File logFile = new File( + "/home/andi/ws/performanceDb/data/production/ondem/ondem01/ap001/1_performance.log"); + + final Grok grok = createGrok(dataDirectory); + + grok.compile(createPattern()); + + try (PerformanceDb db = new PerformanceDb(dataDirectory)) { + + final PerformanceLogs performanceLogs = new PerformanceLogs(); + final Tags tags = Tags.create("pod", "ondem01"); + + final long start = System.nanoTime(); + + performanceLogs.ingest(db, logFile, tags, grok); + + System.out.println("duration: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + } + } + + private static Grok createGrok(final Path dataDirectory) throws GrokException, IOException { + + final File patternsFile = storePatterns(dataDirectory); + + return Grok.create(patternsFile.getAbsolutePath()); + } + + private static File storePatterns(final Path dataDirectory) throws IOException { + + final File file = new File(dataDirectory.toFile(), "patterns"); + try (InputStream stream = Ingestor.class.getClassLoader().getResourceAsStream("org/lucares/grok/patterns")) { + + Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + + return file; + } +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java index f37946f..05b4d52 100644 --- a/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java @@ -1,19 +1,70 @@ package org.lucares.recommind.logs; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.lucares.performance.db.BlockingIterator; +import org.lucares.performance.db.BlockingQueueIterator; import org.lucares.performance.db.Entry; import org.lucares.performance.db.PerformanceDb; import org.lucares.performance.db.Tags; +import io.thekraken.grok.api.Grok; + public class PerformanceLogs { - public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags) { + private final ExecutorService executor = Executors.newCachedThreadPool(); + + public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags, final Grok grok) + throws InterruptedException { final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10); - db.put(queue, tags); + final BlockingIterator iterator = new BlockingQueueIterator<>(queue, Entry.POISON); + final Future future = executor.submit(() -> { + + final GrokToEntryFilter filter = new GrokToEntryFilter(grok); + + boolean result = false; + + try (final BufferedReader reader = new BufferedReader(new FileReader(performanceLog))) { + String line; + while ((line = reader.readLine()) != null) { + final Entry entry = filter.parse(line, tags); + if (entry != null) { + System.out.println(entry); + queue.put(entry); + } + } + + result = true; + } finally { + queue.put(Entry.POISON); + } + return result; + }); + + try { + db.put(iterator); + try { + future.get(10, TimeUnit.MINUTES); + } catch (ExecutionException | TimeoutException e) { + e.printStackTrace(); // TODO @ahr handle this mess + } + } catch (final RuntimeException e) { + future.cancel(true); + } finally { + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.MINUTES); + } } } diff --git a/recommind-logs/src/main/resources/org/lucares/grok/patterns b/recommind-logs/src/main/resources/org/lucares/grok/patterns new file mode 100644 index 0000000..5917f03 --- /dev/null +++ b/recommind-logs/src/main/resources/org/lucares/grok/patterns @@ -0,0 +1,94 @@ +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?/(?>[\w_%!$@:.,-]+|\\.)*)+ +TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) +WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG (?:[\w._/%-]+) +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: +COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} + +# Log Levels +LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) \ No newline at end of file