insert entries for different tags in one stream
This commit is contained in:
@@ -3,7 +3,7 @@ package org.lucares.performance.db;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
|
||||
final class BlockingQueueIterator<E> implements BlockingIterator<E> {
|
||||
public final class BlockingQueueIterator<E> implements BlockingIterator<E> {
|
||||
|
||||
private final BlockingQueue<E> queue;
|
||||
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package org.lucares.performance.db;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.time.temporal.TemporalAccessor;
|
||||
import java.time.temporal.TemporalQuery;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.TimeZone;
|
||||
|
||||
public class DateUtils {
|
||||
@@ -33,14 +36,22 @@ public class DateUtils {
|
||||
return result;
|
||||
}
|
||||
|
||||
public static String format(final Date date) {
|
||||
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH-mm:ss,SSS Z");
|
||||
dateFormat.setTimeZone(TIME_ZONE_UTC);
|
||||
|
||||
return dateFormat.format(date);
|
||||
}
|
||||
|
||||
public static OffsetDateTime nowInUtc() {
|
||||
return OffsetDateTime.now(ZoneOffset.UTC);
|
||||
}
|
||||
|
||||
public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter,
|
||||
final ZoneOffset zoneOffset) {
|
||||
final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
|
||||
|
||||
@Override
|
||||
public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
|
||||
final LocalDate localDate = LocalDate.from(temporal);
|
||||
final LocalTime localTime = LocalTime.from(temporal);
|
||||
return OffsetDateTime.of(localDate, localTime, zoneOffset);
|
||||
}
|
||||
};
|
||||
final OffsetDateTime date = formatter.parse(text, query);
|
||||
return date;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,20 +9,40 @@ import java.util.Comparator;
|
||||
public class Entry {
|
||||
public static final Comparator<Entry> BY_DATE = new EntryByDateComparator();
|
||||
|
||||
/**
|
||||
* A special {@link Entry} that can be used as poison object for
|
||||
* {@link BlockingQueueIterator}.
|
||||
*/
|
||||
public static final Entry POISON = new Entry(0, -1);
|
||||
|
||||
public static final long MAX_VALUE = 0xFF_FF_FF_FFL;
|
||||
|
||||
private final long epochMilli;
|
||||
|
||||
private final long value;
|
||||
|
||||
public Entry(final OffsetDateTime date, final long value) {
|
||||
super();
|
||||
private final Tags tags;
|
||||
|
||||
public Entry(final OffsetDateTime date, final long value, final Tags tags) {
|
||||
this.tags = tags;
|
||||
this.epochMilli = date.toInstant().toEpochMilli();
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
Entry(final long epochMilli, final long value) {
|
||||
super();
|
||||
Entry(final long epochMilli, final long value, final Tags tags) {
|
||||
if (value < 0 || value > MAX_VALUE) {
|
||||
throw new IllegalArgumentException("value must be between 0 and " + MAX_VALUE);
|
||||
}
|
||||
|
||||
this.epochMilli = epochMilli;
|
||||
this.value = value;
|
||||
this.tags = tags;
|
||||
}
|
||||
|
||||
private Entry(final long epochMilli, final long value) {
|
||||
this.epochMilli = epochMilli;
|
||||
this.value = value;
|
||||
this.tags = null;
|
||||
}
|
||||
|
||||
public OffsetDateTime getDate() {
|
||||
@@ -38,10 +58,14 @@ public class Entry {
|
||||
return epochMilli;
|
||||
}
|
||||
|
||||
public Tags getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
final OffsetDateTime date = getDate();
|
||||
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value;
|
||||
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -49,6 +73,7 @@ public class Entry {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32));
|
||||
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
|
||||
result = prime * result + (int) (value ^ (value >>> 32));
|
||||
return result;
|
||||
}
|
||||
@@ -64,9 +89,13 @@ public class Entry {
|
||||
final Entry other = (Entry) obj;
|
||||
if (epochMilli != other.epochMilli)
|
||||
return false;
|
||||
if (tags == null) {
|
||||
if (other.tags != null)
|
||||
return false;
|
||||
} else if (!tags.equals(other.tags))
|
||||
return false;
|
||||
if (value != other.value)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import java.util.Comparator;
|
||||
|
||||
public class PdbFileByTimeAsc implements Comparator<PdbFile> {
|
||||
|
||||
public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc();
|
||||
|
||||
@Override
|
||||
public int compare(final PdbFile o1, final PdbFile o2) {
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
|
||||
|
||||
private final Queue<PdbFile> pdbFiles;
|
||||
private PdbReader reader;
|
||||
private PdbFile currentPdbFile;
|
||||
|
||||
public EntrySupplier(final Collection<PdbFile> pdbFiles) {
|
||||
super();
|
||||
@@ -30,14 +31,15 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
|
||||
if (reader == null) {
|
||||
nextFile();
|
||||
}
|
||||
final Optional<Entry> optionalEntry = reader.readEntry();
|
||||
final Optional<Entry> optionalEntry = reader.readEntry(currentPdbFile.getTags());
|
||||
|
||||
return optionalEntry.orElseGet(() -> {
|
||||
nextFile();
|
||||
if (reader == null) {
|
||||
return null;
|
||||
} else {
|
||||
return reader.readEntry().orElse(null);
|
||||
final Tags tags = currentPdbFile.getTags();
|
||||
return reader.readEntry(tags).orElse(null);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -51,16 +53,16 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
|
||||
}
|
||||
|
||||
while (!pdbFiles.isEmpty()) {
|
||||
final PdbFile pdbFile = pdbFiles.poll();
|
||||
currentPdbFile = pdbFiles.poll();
|
||||
try {
|
||||
if (pdbFile.getFile().length() > 0) {
|
||||
reader = new PdbReader(pdbFile);
|
||||
if (currentPdbFile.getFile().length() > 0) {
|
||||
reader = new PdbReader(currentPdbFile);
|
||||
break;
|
||||
} else {
|
||||
LOGGER.info("ignoring empty file " + pdbFile);
|
||||
LOGGER.info("ignoring empty file " + currentPdbFile);
|
||||
}
|
||||
} catch (final FileNotFoundException e) {
|
||||
LOGGER.log(Level.WARNING, "the pdbFile " + pdbFile.getFile() + " is missing", e);
|
||||
LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getFile() + " is missing", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package org.lucares.performance.db;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
class PdbFileUtils {
|
||||
static TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
|
||||
|
||||
try (PdbReader reader = new PdbReader(pdbFile)) {
|
||||
if (reader.canSeekTail(2)) {
|
||||
reader.seekTail(2);
|
||||
final OffsetDateTime lastWrittenDate = reader.readDate();
|
||||
|
||||
return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo());
|
||||
} else {
|
||||
return pdbFile.getTimeRange();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ class PdbReader implements AutoCloseable {
|
||||
}
|
||||
}
|
||||
|
||||
public Optional<Entry> readEntry() throws ReadRuntimeException {
|
||||
public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
|
||||
final long epochMilli = readEpochMilli();
|
||||
if (epochMilli < 0) {
|
||||
return Optional.empty();
|
||||
@@ -193,7 +193,7 @@ class PdbReader implements AutoCloseable {
|
||||
if (value < 0) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(new Entry(epochMilli, value));
|
||||
return Optional.of(new Entry(epochMilli, value, tags));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
package org.lucares.performance.db;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
||||
class PdbWriter implements AutoCloseable {
|
||||
|
||||
private static final long MAX_VALUE = 0xFF_FF_FF_FFL;
|
||||
private static final boolean APPEND = true;
|
||||
private final OutputStream outputStream;
|
||||
private final PdbFile pdbFile;
|
||||
private long minimalEpochMilli;
|
||||
|
||||
PdbWriter(final PdbFile pdbFile) throws FileNotFoundException {
|
||||
PdbWriter(final PdbFile pdbFile) throws IOException {
|
||||
this.pdbFile = pdbFile;
|
||||
this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND));
|
||||
|
||||
if (pdbFile.getFile().exists() && pdbFile.getFile().length() > 0) {
|
||||
final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
|
||||
minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli();
|
||||
} else {
|
||||
minimalEpochMilli = pdbFile.getTimeRange().getFrom().toInstant().toEpochMilli();
|
||||
}
|
||||
}
|
||||
|
||||
public PdbFile getFile() {
|
||||
@@ -27,21 +33,29 @@ class PdbWriter implements AutoCloseable {
|
||||
}
|
||||
|
||||
void write(final long epochMilli, final long value) throws WriteException {
|
||||
final long offsetEpochMill = pdbFile.getOffsetInEpochMilli();
|
||||
final long adjustedValue = epochMilli - offsetEpochMill;
|
||||
final long offsetEpochMilli = pdbFile.getOffsetInEpochMilli();
|
||||
final long adjustedValue = epochMilli - offsetEpochMilli;
|
||||
assertValueInRange(adjustedValue);
|
||||
assertValueInRange(value);
|
||||
assertEpochMilliInRange(epochMilli);
|
||||
|
||||
write(adjustedValue);
|
||||
write(value);
|
||||
minimalEpochMilli = epochMilli;
|
||||
}
|
||||
|
||||
private void assertEpochMilliInRange(final long epochMilli) {
|
||||
if (epochMilli < minimalEpochMilli) {
|
||||
throw new IllegalArgumentException("value must not be smaller than: " + minimalEpochMilli);
|
||||
}
|
||||
}
|
||||
|
||||
private void assertValueInRange(final long value) {
|
||||
if (value < 0) {
|
||||
throw new IllegalArgumentException("value must not be negative: " + value);
|
||||
}
|
||||
if (value > MAX_VALUE) {
|
||||
throw new IllegalArgumentException("max value is " + MAX_VALUE + " value was: " + value);
|
||||
if (value > Entry.MAX_VALUE) {
|
||||
throw new IllegalArgumentException("max value is " + Entry.MAX_VALUE + " value was: " + value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package org.lucares.performance.db;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
public class PdbWriterManager implements AutoCloseable {
|
||||
|
||||
private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName());
|
||||
|
||||
private final static class Key {
|
||||
private final Tags tags;
|
||||
private final Day day;
|
||||
|
||||
public Key(final Tags tags, final OffsetDateTime date) {
|
||||
super();
|
||||
this.tags = tags;
|
||||
this.day = new Day(date);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + ((day == null) ? 0 : day.hashCode());
|
||||
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
|
||||
return result;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (this == obj)
|
||||
return true;
|
||||
if (obj == null)
|
||||
return false;
|
||||
if (getClass() != obj.getClass())
|
||||
return false;
|
||||
final Key other = (Key) obj;
|
||||
if (day == null) {
|
||||
if (other.day != null)
|
||||
return false;
|
||||
} else if (!day.equals(other.day))
|
||||
return false;
|
||||
if (tags == null) {
|
||||
if (other.tags != null)
|
||||
return false;
|
||||
} else if (!tags.equals(other.tags))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public interface PdbWriterSupplier {
|
||||
public PdbWriter supply(Tags tags, OffsetDateTime date);
|
||||
}
|
||||
|
||||
final Map<Key, PdbWriter> map = new HashMap<>();
|
||||
|
||||
private final PdbWriterSupplier supplier;
|
||||
|
||||
public PdbWriterManager(final PdbWriterSupplier supplier) {
|
||||
this.supplier = supplier;
|
||||
}
|
||||
|
||||
public PdbWriter get(final Tags tags, final OffsetDateTime date) {
|
||||
|
||||
final Key key = new Key(tags, date);
|
||||
if (!map.containsKey(key)) {
|
||||
final PdbWriter writer = supplier.supply(tags, date);
|
||||
put(tags, date, writer);
|
||||
}
|
||||
return map.get(key);
|
||||
}
|
||||
|
||||
public PdbWriter put(final Tags tags, final OffsetDateTime date, final PdbWriter pdbWriter) {
|
||||
final Key key = new Key(tags, date);
|
||||
return map.put(key, pdbWriter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
for (final PdbWriter writer : map.values()) {
|
||||
try {
|
||||
writer.close();
|
||||
} catch (final IOException e) {
|
||||
LOGGER.log(Level.WARNING, e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@ import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
|
||||
|
||||
import liquibase.exception.LiquibaseException;
|
||||
|
||||
public class PerformanceDb implements AutoCloseable {
|
||||
@@ -26,38 +28,54 @@ public class PerformanceDb implements AutoCloseable {
|
||||
tagsToFile = new TagsToFile(dataDirectory);
|
||||
}
|
||||
|
||||
public void put(final OffsetDateTime date, final long value, final Tags tags) throws WriteException {
|
||||
put(new Entry(date, value), tags);
|
||||
public void put(final Entry entry) throws WriteException {
|
||||
put(Arrays.asList(entry));
|
||||
}
|
||||
|
||||
public void put(final Entry entry, final Tags tags) throws WriteException {
|
||||
put(Arrays.asList(entry), tags);
|
||||
public void put(final Iterable<Entry> entries) throws WriteException {
|
||||
put(entries.iterator());
|
||||
}
|
||||
|
||||
public void put(final Iterable<Entry> entries, final Tags tags) throws WriteException {
|
||||
put(entries.iterator(), tags);
|
||||
}
|
||||
|
||||
public void put(final BlockingQueue<Entry> entries, final Entry poisonObject, final Tags tags)
|
||||
throws WriteException {
|
||||
public void put(final BlockingQueue<Entry> entries, final Entry poisonObject) throws WriteException {
|
||||
final BlockingQueueIterator<Entry> iterator = new BlockingQueueIterator<>(entries, poisonObject);
|
||||
put(iterator, tags);
|
||||
put(iterator);
|
||||
}
|
||||
|
||||
public void put(final Iterator<Entry> entries, final Tags tags) throws WriteException {
|
||||
public void put(final Iterator<Entry> entries) throws WriteException {
|
||||
|
||||
final BlockingIteratorIterator<Entry> iterator = new BlockingIteratorIterator<>(entries);
|
||||
put(iterator, tags);
|
||||
put(iterator);
|
||||
}
|
||||
|
||||
public void put(final BlockingIterator<Entry> entries, final Tags tags) throws WriteException {
|
||||
private static class WriterSupplier implements PdbWriterSupplier {
|
||||
|
||||
private final TagsToFile tagsToFile;
|
||||
|
||||
public WriterSupplier(final TagsToFile tagsToFile) {
|
||||
super();
|
||||
this.tagsToFile = tagsToFile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PdbWriter supply(final Tags tags, final OffsetDateTime date) {
|
||||
try {
|
||||
final PdbFile pdbFile = tagsToFile.getFile(date, tags);
|
||||
|
||||
final PdbWriter writer = new PdbWriter(pdbFile);
|
||||
return writer;
|
||||
} catch (final IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void put(final BlockingIterator<Entry> entries) throws WriteException {
|
||||
|
||||
final long start = System.nanoTime();
|
||||
final double timeSpendInWrite = 0.0;
|
||||
long count = 0;
|
||||
PdbWriter writer = null;
|
||||
PdbFile pdbFile = null;
|
||||
try {
|
||||
|
||||
try (final PdbWriterManager manager = new PdbWriterManager(new WriterSupplier(tagsToFile));) {
|
||||
while (true) {
|
||||
final Optional<Entry> entryOptional = entries.next();
|
||||
if (!entryOptional.isPresent()) {
|
||||
@@ -67,29 +85,10 @@ public class PerformanceDb implements AutoCloseable {
|
||||
|
||||
final long epochMilli = entry.getEpochMilli();
|
||||
final long value = entry.getValue();
|
||||
final Tags tags = entry.getTags();
|
||||
final OffsetDateTime date = entry.getDate();
|
||||
|
||||
if (pdbFile == null //
|
||||
|| !pdbFile.getTimeRange().inRange(epochMilli)) // TODO
|
||||
// @ahr
|
||||
// correct
|
||||
// would be
|
||||
// to check
|
||||
// if the
|
||||
// date is
|
||||
// in the
|
||||
// available
|
||||
// range
|
||||
{
|
||||
final OffsetDateTime date = entry.getDate();
|
||||
pdbFile = tagsToFile.getFile(date, tags);
|
||||
}
|
||||
|
||||
if (writer == null || !writer.getFile().equals(pdbFile)) {
|
||||
if (writer != null) {
|
||||
writer.close();
|
||||
}
|
||||
writer = new PdbWriter(pdbFile);
|
||||
}
|
||||
final PdbWriter writer = manager.get(tags, date);
|
||||
|
||||
writer.write(epochMilli, value);
|
||||
count++;
|
||||
@@ -98,17 +97,7 @@ public class PerformanceDb implements AutoCloseable {
|
||||
} catch (final InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
LOGGER.info("Thread was interrupted. Aborting exectution.");
|
||||
} catch (final IOException e) {
|
||||
throw new WriteException(e);
|
||||
} finally {
|
||||
if (writer != null) {
|
||||
try {
|
||||
writer.close();
|
||||
} catch (final IOException e) {
|
||||
throw new WriteException(e);
|
||||
}
|
||||
}
|
||||
|
||||
final double duration = (System.nanoTime() - start) / 1_000_000.0;
|
||||
LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite
|
||||
+ " were spend in write");
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.lucares.performance.db;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
@@ -9,33 +10,55 @@ import java.util.TreeSet;
|
||||
import java.util.function.BiConsumer;
|
||||
|
||||
public class Tags {
|
||||
|
||||
private static final Tags EMPTY = new Tags();
|
||||
|
||||
private final Map<String, String> tags = new HashMap<>();
|
||||
|
||||
public Tags() {
|
||||
private Tags() {
|
||||
super();
|
||||
}
|
||||
|
||||
public Tags(final String key, final String value) {
|
||||
super();
|
||||
tags.put(key, value);
|
||||
private Tags(final Map<String, String> tags) {
|
||||
this.tags.putAll(tags);
|
||||
}
|
||||
|
||||
public Tags(final String key1, final String value1, final String key2, final String value2) {
|
||||
super();
|
||||
public static Tags create() {
|
||||
return EMPTY;
|
||||
}
|
||||
|
||||
public static Tags create(final String key1, final String value1, final String key2, final String value2) {
|
||||
final Map<String, String> tags = new HashMap<>(2);
|
||||
tags.put(key1, value1);
|
||||
tags.put(key2, value2);
|
||||
return new Tags(tags); // TODO @ahr cache them
|
||||
}
|
||||
|
||||
public Tags(final String key1, final String value1, final String key2, final String value2, final String key3,
|
||||
final String value3) {
|
||||
super();
|
||||
public static Tags create(final String key1, final String value1) {
|
||||
final Map<String, String> tags = new HashMap<>(1);
|
||||
tags.put(key1, value1);
|
||||
tags.put(key2, value2);
|
||||
tags.put(key3, value3);
|
||||
return new Tags(tags);
|
||||
}
|
||||
|
||||
public void put(final String key, final String value) {
|
||||
tags.put(key, value);
|
||||
public Tags copyAdd(final String key, final String value) {
|
||||
Objects.requireNonNull(key, "key must not be null");
|
||||
Objects.requireNonNull(value, "value must not be null");
|
||||
|
||||
final Map<String, String> newTags = new HashMap<>(tags);
|
||||
newTags.put(key.intern(), value.intern());
|
||||
|
||||
return new Tags(newTags);
|
||||
}
|
||||
|
||||
public Tags copyAddIfNotNull(final String key, final String value) {
|
||||
|
||||
final Tags result;
|
||||
if (value != null) {
|
||||
result = copyAdd(key, value);
|
||||
} else {
|
||||
result = this;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public Optional<String> getValue(final String key) {
|
||||
@@ -85,7 +108,7 @@ public class Tags {
|
||||
|
||||
public String abbreviatedRepresentation() {
|
||||
final StringBuilder result = new StringBuilder();
|
||||
final int maxLength = 200;
|
||||
final int maxLength = 500;
|
||||
|
||||
final SortedSet<String> keys = new TreeSet<>(tags.keySet());
|
||||
|
||||
@@ -105,4 +128,5 @@ public class Tags {
|
||||
private static String substr(final String s, final int maxLength) {
|
||||
return s.substring(0, Math.min(maxLength, s.length()));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
|
||||
// TODO @ahr ludb should handle unknown fields better
|
||||
}
|
||||
|
||||
Collections.sort(result, new PdbFileByTimeAsc());
|
||||
Collections.sort(result, PdbFileByTimeAsc.INSTANCE);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -77,13 +77,13 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
|
||||
}
|
||||
|
||||
private Tags toTags(final Document document) {
|
||||
final Tags tagsOfFile = new Tags();
|
||||
Tags tagsOfFile = Tags.create();
|
||||
|
||||
for (final String key : document.getProperties().keySet()) {
|
||||
|
||||
if (Fields.isPrefixedKey(key)) {
|
||||
final String value = document.getPropertyString(key);
|
||||
tagsOfFile.put(Fields.stripPrefix(key), value);
|
||||
tagsOfFile = tagsOfFile.copyAdd(Fields.stripPrefix(key), value);
|
||||
}
|
||||
}
|
||||
return tagsOfFile;
|
||||
@@ -99,7 +99,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
|
||||
final boolean inRange = pdbFile.getTimeRange().inRange(date);
|
||||
|
||||
if (inRange) {
|
||||
final TimeRange availableTimeRange = getAvailableTimeRange(pdbFile);
|
||||
final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
|
||||
|
||||
if (availableTimeRange.inRange(date)) {
|
||||
preResult.add(pdbFile);
|
||||
@@ -119,20 +119,6 @@ public class TagsToFile implements AutoCloseable, CollectionUtils {
|
||||
return result;
|
||||
}
|
||||
|
||||
private TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
|
||||
|
||||
try (PdbReader reader = new PdbReader(pdbFile)) {
|
||||
if (reader.canSeekTail(2)) {
|
||||
reader.seekTail(2);
|
||||
final OffsetDateTime lastWrittenDate = reader.readDate();
|
||||
|
||||
return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo());
|
||||
} else {
|
||||
return pdbFile.getTimeRange();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) {
|
||||
final File file;
|
||||
PdbFile result;
|
||||
|
||||
@@ -40,17 +40,18 @@ public class PdbReaderWriterTest {
|
||||
public void testWriteRead(final long value) throws Exception {
|
||||
|
||||
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
|
||||
final PdbFile pdbFile = PdbFile.today(file, new Tags());
|
||||
final Tags tags = Tags.create();
|
||||
final PdbFile pdbFile = PdbFile.today(file, tags);
|
||||
final OffsetDateTime now = OffsetDateTime.now(); // TODO @ahr might fail
|
||||
// at midnight
|
||||
final Entry entry = new Entry(now, value);
|
||||
final Entry entry = new Entry(now, value, tags);
|
||||
|
||||
try (PdbWriter writer = new PdbWriter(pdbFile)) {
|
||||
writer.write(entry);
|
||||
}
|
||||
|
||||
try (final PdbReader reader = new PdbReader(pdbFile)) {
|
||||
final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
|
||||
final Entry actual = reader.readEntry(tags).orElseThrow(() -> new AssertionError());
|
||||
|
||||
Assert.assertEquals(actual, entry);
|
||||
}
|
||||
@@ -59,7 +60,7 @@ public class PdbReaderWriterTest {
|
||||
public void testSeekTail() throws Exception {
|
||||
|
||||
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
|
||||
final PdbFile pdbFile = PdbFile.today(file, new Tags());
|
||||
final PdbFile pdbFile = PdbFile.today(file, Tags.create());
|
||||
|
||||
try (PdbWriter writer = new PdbWriter(pdbFile)) {
|
||||
writer.write(1);
|
||||
@@ -86,7 +87,7 @@ public class PdbReaderWriterTest {
|
||||
public void testSeek() throws Exception {
|
||||
|
||||
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
|
||||
final PdbFile pdbFile = PdbFile.today(file, new Tags());
|
||||
final PdbFile pdbFile = PdbFile.today(file, Tags.create());
|
||||
|
||||
try (PdbWriter writer = new PdbWriter(pdbFile)) {
|
||||
writer.write(1);
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package org.lucares.performance.db;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
@Test
|
||||
public class PdbWriterManagerTest {
|
||||
|
||||
private Path dataDirectory;
|
||||
|
||||
@BeforeMethod
|
||||
public void beforeMethod() throws IOException {
|
||||
dataDirectory = Files.createTempDirectory("pdb");
|
||||
}
|
||||
|
||||
@AfterMethod
|
||||
public void afterMethod() throws IOException {
|
||||
FileUtils.delete(dataDirectory);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testManager() throws Exception {
|
||||
|
||||
final PdbWriterSupplier supplier = (tags, date) -> {
|
||||
File file;
|
||||
try {
|
||||
file = Files.createTempFile(dataDirectory, "pdb", ".data").toFile();
|
||||
return new PdbWriter(new PdbFile(new Day(date), file, tags));
|
||||
} catch (final IOException e) {
|
||||
throw new AssertionError(e.getMessage(), e);
|
||||
}
|
||||
};
|
||||
final Tags tagsA = Tags.create("key", "A");
|
||||
final Tags tagsB = Tags.create("key", "B");
|
||||
|
||||
try (PdbWriterManager manager = new PdbWriterManager(supplier)) {
|
||||
|
||||
final OffsetDateTime date = OffsetDateTime.now();
|
||||
|
||||
final PdbWriter firstWriterForTagsA = manager.get(tagsA, date);
|
||||
final PdbWriter secondWriterForTagsA = manager.get(tagsA, date);
|
||||
final PdbWriter firstWriterForTagsB = manager.get(tagsB, date);
|
||||
|
||||
Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
|
||||
Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -38,14 +38,14 @@ public class PerformanceDbTest {
|
||||
try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
|
||||
final OffsetDateTime date = DateUtils.nowInUtc();
|
||||
final long value = 1;
|
||||
final Tags tags = new Tags("myKey", "myValue");
|
||||
performanceDb.put(date, value, tags);
|
||||
final Tags tags = Tags.create("myKey", "myValue");
|
||||
performanceDb.put(new Entry(date, value, tags));
|
||||
|
||||
final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
|
||||
|
||||
Assert.assertEquals(stream.size(), 1);
|
||||
|
||||
Assert.assertEquals(stream.get(0), new Entry(date, value));
|
||||
Assert.assertEquals(stream.get(0), new Entry(date, value, tags));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,28 +56,28 @@ public class PerformanceDbTest {
|
||||
final OffsetDateTime dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56);
|
||||
final long valueOne = 1;
|
||||
final long valueTwo = 2;
|
||||
final Tags tags = new Tags("myKey", "myValue");
|
||||
final Tags tags = Tags.create("myKey", "myValue");
|
||||
|
||||
performanceDb.put(dayOne, valueOne, tags);
|
||||
performanceDb.put(dayTwo, valueTwo, tags);
|
||||
performanceDb.put(new Entry(dayOne, valueOne, tags));
|
||||
performanceDb.put(new Entry(dayTwo, valueTwo, tags));
|
||||
|
||||
final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
|
||||
|
||||
Assert.assertEquals(stream.size(), 2);
|
||||
|
||||
Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne));
|
||||
Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo));
|
||||
Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne, tags));
|
||||
Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo, tags));
|
||||
}
|
||||
}
|
||||
|
||||
private List<Entry> generateEntries(final TimeRange timeRange, final long n, final int addToDate) {
|
||||
private List<Entry> generateEntries(final TimeRange timeRange, final long n, final int addToDate, final Tags tags) {
|
||||
final List<Entry> result = new ArrayList<>();
|
||||
final long differenceInMs = timeRange.duration().toMillis() / n;
|
||||
long currentTime = timeRange.getFrom().toInstant().toEpochMilli();
|
||||
|
||||
for (long i = 0; i < n; i++) {
|
||||
final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE);
|
||||
result.add(new Entry(currentTime + addToDate, value));
|
||||
result.add(new Entry(currentTime + addToDate, value, tags));
|
||||
|
||||
currentTime += differenceInMs;
|
||||
}
|
||||
@@ -91,13 +91,13 @@ public class PerformanceDbTest {
|
||||
final TimeRange timeRange = TimeRange.ofDay(OffsetDateTime.now(ZoneOffset.UTC));
|
||||
final long numberOfEntries = 2;
|
||||
|
||||
final Tags tags = new Tags("myKey", "one");
|
||||
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0);
|
||||
final Tags tags = Tags.create("myKey", "one");
|
||||
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0, tags);
|
||||
|
||||
printEntries(entries, "");
|
||||
|
||||
for (final Entry entry : entries) {
|
||||
performanceDb.put(entry, tags);
|
||||
performanceDb.put(entry);
|
||||
}
|
||||
|
||||
final List<Entry> actualEntries = performanceDb.getAsList(tags);
|
||||
@@ -121,21 +121,21 @@ public class PerformanceDbTest {
|
||||
final TimeRange timeRange = new TimeRange(from, to);
|
||||
final long numberOfEntries = timeRange.duration().toHours();
|
||||
|
||||
final Tags tagsCommon = new Tags("commonKey", "commonValue");
|
||||
final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue");
|
||||
final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries, 1);
|
||||
final Tags tagsCommon = Tags.create("commonKey", "commonValue");
|
||||
final Tags tagsOne = Tags.create("myKey", "one", "commonKey", "commonValue");
|
||||
final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries, 1, tagsOne);
|
||||
printEntries(entriesOne, "one");
|
||||
performanceDb.put(entriesOne, tagsOne);
|
||||
performanceDb.put(entriesOne);
|
||||
|
||||
final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue");
|
||||
final List<Entry> entriesTwo = generateEntries(timeRange, numberOfEntries, 2);
|
||||
final Tags tagsTwo = Tags.create("myKey", "two", "commonKey", "commonValue");
|
||||
final List<Entry> entriesTwo = generateEntries(timeRange, numberOfEntries, 2, tagsTwo);
|
||||
printEntries(entriesTwo, "two");
|
||||
performanceDb.put(entriesTwo, tagsTwo);
|
||||
performanceDb.put(entriesTwo);
|
||||
|
||||
final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue");
|
||||
final List<Entry> entriesThree = generateEntries(timeRange, numberOfEntries, 3);
|
||||
final Tags tagsThree = Tags.create("myKey", "three", "commonKey", "commonValue");
|
||||
final List<Entry> entriesThree = generateEntries(timeRange, numberOfEntries, 3, tagsThree);
|
||||
printEntries(entriesThree, "three");
|
||||
performanceDb.put(entriesThree, tagsThree);
|
||||
performanceDb.put(entriesThree);
|
||||
|
||||
final List<Entry> actualEntriesOne = performanceDb.getAsList(tagsOne);
|
||||
Assert.assertEquals(actualEntriesOne, entriesOne);
|
||||
|
||||
@@ -30,7 +30,7 @@ public class TagsToFilesTest {
|
||||
try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) {
|
||||
|
||||
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
|
||||
final Tags tags = new Tags("myKey", "myValue");
|
||||
final Tags tags = Tags.create("myKey", "myValue");
|
||||
|
||||
final PdbFile newFileForTags = tagsToFile.getFile(date, tags);
|
||||
final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package org.lucares.recommind.logs;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Map;
|
||||
|
||||
import org.lucares.performance.db.DateUtils;
|
||||
import org.lucares.performance.db.Entry;
|
||||
import org.lucares.performance.db.Tags;
|
||||
|
||||
import io.thekraken.grok.api.Grok;
|
||||
import io.thekraken.grok.api.Match;
|
||||
|
||||
public class GrokToEntryFilter {
|
||||
|
||||
private final Grok grok;
|
||||
|
||||
private final static DateTimeFormatter FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS");
|
||||
|
||||
public GrokToEntryFilter(final Grok grok) {
|
||||
this.grok = grok;
|
||||
}
|
||||
|
||||
public Entry parse(final String singleLineOfLog, final Tags tags) {
|
||||
final Match gm = grok.match(singleLineOfLog);
|
||||
gm.captures();
|
||||
final Map<String, Object> map = gm.toMap();
|
||||
|
||||
final String timestamp = (String) map.get("timestamp");
|
||||
final String method = (String) map.get("method");
|
||||
final String project = (String) map.get("project");
|
||||
final Long value = (Long) map.get("duration");
|
||||
|
||||
final Entry result;
|
||||
if (timestamp == null || method == null || value == null) {
|
||||
result = null;
|
||||
} else {
|
||||
|
||||
final OffsetDateTime date = DateUtils.parseAtZoneOffset(timestamp, FORMAT, ZoneOffset.UTC);
|
||||
|
||||
Tags entryTags = tags;
|
||||
entryTags = entryTags.copyAddIfNotNull("method", method);
|
||||
entryTags = entryTags.copyAddIfNotNull("project", lowerCase(project));
|
||||
|
||||
result = new Entry(date, value, entryTags);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String lowerCase(final String s) {
|
||||
return s != null ? s.toLowerCase().intern() : s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package org.lucares.recommind.logs;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
|
||||
import org.lucares.performance.db.PerformanceDb;
|
||||
import org.lucares.performance.db.Tags;
|
||||
|
||||
import io.thekraken.grok.api.Grok;
|
||||
import io.thekraken.grok.api.exception.GrokException;
|
||||
import liquibase.exception.LiquibaseException;
|
||||
|
||||
public class Ingestor {
|
||||
|
||||
public static String createPattern() {
|
||||
// "%{TIMESTAMP_ISO8601:timestamp}\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?<logger>(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null
|
||||
// - Executed
|
||||
// %{NOTSPACE:method} in %{NUMBER:duration} ms %{WORD:status}.
|
||||
// \\[.*?(?:project=%{WORD:project})?"
|
||||
final String time = "%{TIMESTAMP_ISO8601:timestamp}";
|
||||
final String stuff = "\\s+\\[.*\\]+\\s+%{LOGLEVEL}\\s+(?<logger>(?:[a-zA-Z0-9-]+\\.)*[A-Za-z0-9$]+)\\s+null - Executed ";
|
||||
final String method = "%{NOTSPACE:method}";
|
||||
final String duration = "%{NUMBER:duration:long}";
|
||||
final String status = "%{WORD:status}";
|
||||
final String project = "(?:project=%{WORD:project})?";
|
||||
return time + stuff + method + " in " + duration + " ms " + status + ". \\[.*?" + project;
|
||||
// return time;
|
||||
|
||||
}
|
||||
|
||||
public static void main(final String[] args) throws LiquibaseException, Exception {
|
||||
final Path dataDirectory = Files.createTempDirectory("ingestor");
|
||||
final File logFile = new File(
|
||||
"/home/andi/ws/performanceDb/data/production/ondem/ondem01/ap001/1_performance.log");
|
||||
|
||||
final Grok grok = createGrok(dataDirectory);
|
||||
|
||||
grok.compile(createPattern());
|
||||
|
||||
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
|
||||
|
||||
final PerformanceLogs performanceLogs = new PerformanceLogs();
|
||||
final Tags tags = Tags.create("pod", "ondem01");
|
||||
|
||||
final long start = System.nanoTime();
|
||||
|
||||
performanceLogs.ingest(db, logFile, tags, grok);
|
||||
|
||||
System.out.println("duration: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
|
||||
}
|
||||
}
|
||||
|
||||
private static Grok createGrok(final Path dataDirectory) throws GrokException, IOException {
|
||||
|
||||
final File patternsFile = storePatterns(dataDirectory);
|
||||
|
||||
return Grok.create(patternsFile.getAbsolutePath());
|
||||
}
|
||||
|
||||
private static File storePatterns(final Path dataDirectory) throws IOException {
|
||||
|
||||
final File file = new File(dataDirectory.toFile(), "patterns");
|
||||
try (InputStream stream = Ingestor.class.getClassLoader().getResourceAsStream("org/lucares/grok/patterns")) {
|
||||
|
||||
Files.copy(stream, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,70 @@
|
||||
package org.lucares.recommind.logs;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
import org.lucares.performance.db.BlockingIterator;
|
||||
import org.lucares.performance.db.BlockingQueueIterator;
|
||||
import org.lucares.performance.db.Entry;
|
||||
import org.lucares.performance.db.PerformanceDb;
|
||||
import org.lucares.performance.db.Tags;
|
||||
|
||||
import io.thekraken.grok.api.Grok;
|
||||
|
||||
public class PerformanceLogs {
|
||||
|
||||
public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags) {
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags, final Grok grok)
|
||||
throws InterruptedException {
|
||||
|
||||
final ArrayBlockingQueue<Entry> queue = new ArrayBlockingQueue<>(10);
|
||||
|
||||
db.put(queue, tags);
|
||||
final BlockingIterator<Entry> iterator = new BlockingQueueIterator<>(queue, Entry.POISON);
|
||||
|
||||
final Future<Boolean> future = executor.submit(() -> {
|
||||
|
||||
final GrokToEntryFilter filter = new GrokToEntryFilter(grok);
|
||||
|
||||
boolean result = false;
|
||||
|
||||
try (final BufferedReader reader = new BufferedReader(new FileReader(performanceLog))) {
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
final Entry entry = filter.parse(line, tags);
|
||||
if (entry != null) {
|
||||
System.out.println(entry);
|
||||
queue.put(entry);
|
||||
}
|
||||
}
|
||||
|
||||
result = true;
|
||||
} finally {
|
||||
queue.put(Entry.POISON);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
try {
|
||||
db.put(iterator);
|
||||
try {
|
||||
future.get(10, TimeUnit.MINUTES);
|
||||
} catch (ExecutionException | TimeoutException e) {
|
||||
e.printStackTrace(); // TODO @ahr handle this mess
|
||||
}
|
||||
} catch (final RuntimeException e) {
|
||||
future.cancel(true);
|
||||
} finally {
|
||||
executor.shutdown();
|
||||
executor.awaitTermination(10, TimeUnit.MINUTES);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
94
recommind-logs/src/main/resources/org/lucares/grok/patterns
Normal file
94
recommind-logs/src/main/resources/org/lucares/grok/patterns
Normal file
@@ -0,0 +1,94 @@
|
||||
USERNAME [a-zA-Z0-9._-]+
|
||||
USER %{USERNAME}
|
||||
INT (?:[+-]?(?:[0-9]+))
|
||||
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
|
||||
NUMBER (?:%{BASE10NUM})
|
||||
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
|
||||
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
|
||||
|
||||
POSINT \b(?:[1-9][0-9]*)\b
|
||||
NONNEGINT \b(?:[0-9]+)\b
|
||||
WORD \b\w+\b
|
||||
NOTSPACE \S+
|
||||
SPACE \s*
|
||||
DATA .*?
|
||||
GREEDYDATA .*
|
||||
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
|
||||
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
|
||||
|
||||
# Networking
|
||||
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
|
||||
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
|
||||
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
|
||||
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
|
||||
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
|
||||
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
|
||||
IP (?:%{IPV6}|%{IPV4})
|
||||
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
|
||||
HOST %{HOSTNAME}
|
||||
IPORHOST (?:%{HOSTNAME}|%{IP})
|
||||
HOSTPORT %{IPORHOST}:%{POSINT}
|
||||
|
||||
# paths
|
||||
PATH (?:%{UNIXPATH}|%{WINPATH})
|
||||
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
|
||||
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
|
||||
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
|
||||
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
|
||||
URIHOST %{IPORHOST}(?::%{POSINT:port})?
|
||||
# uripath comes loosely from RFC1738, but mostly from what Firefox
|
||||
# doesn't turn into %XX
|
||||
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
|
||||
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
|
||||
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
|
||||
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
|
||||
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
|
||||
|
||||
# Months: January, Feb, 3, 03, 12, December
|
||||
MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
|
||||
MONTHNUM (?:0?[1-9]|1[0-2])
|
||||
MONTHNUM2 (?:0[1-9]|1[0-2])
|
||||
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
|
||||
|
||||
# Days: Monday, Tue, Thu, etc...
|
||||
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
|
||||
|
||||
# Years?
|
||||
YEAR (?>\d\d){1,2}
|
||||
HOUR (?:2[0123]|[01]?[0-9])
|
||||
MINUTE (?:[0-5][0-9])
|
||||
# '60' is a leap second in most time standards and thus is valid.
|
||||
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
|
||||
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
|
||||
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
|
||||
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
|
||||
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
|
||||
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
|
||||
ISO8601_SECOND (?:%{SECOND}|60)
|
||||
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
|
||||
DATE %{DATE_US}|%{DATE_EU}
|
||||
DATESTAMP %{DATE}[- ]%{TIME}
|
||||
TZ (?:[PMCE][SD]T|UTC)
|
||||
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
|
||||
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
|
||||
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
|
||||
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
|
||||
|
||||
# Syslog Dates: Month Day HH:MM:SS
|
||||
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
|
||||
PROG (?:[\w._/%-]+)
|
||||
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
|
||||
SYSLOGHOST %{IPORHOST}
|
||||
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
|
||||
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
|
||||
|
||||
# Shortcuts
|
||||
QS %{QUOTEDSTRING}
|
||||
|
||||
# Log formats
|
||||
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
|
||||
COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
|
||||
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
|
||||
|
||||
# Log Levels
|
||||
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
|
||||
Reference in New Issue
Block a user