diff --git a/performanceDb/build.gradle b/performanceDb/build.gradle index a7fa47a..7121f16 100644 --- a/performanceDb/build.gradle +++ b/performanceDb/build.gradle @@ -3,5 +3,6 @@ dependencies { //compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5' compile 'org.lucares:ludb:1.0.20161002111352' //compile 'commons-io:commons-io:2.5' + //compile 'joda-time:joda-time:2.9.6' } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BlockingIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/BlockingIterator.java new file mode 100644 index 0000000..22202fd --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/BlockingIterator.java @@ -0,0 +1,8 @@ +package org.lucares.performance.db; + +import java.util.Optional; + +public interface BlockingIterator { + + public Optional next() throws InterruptedException; +} \ No newline at end of file diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BlockingIteratorIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/BlockingIteratorIterator.java new file mode 100644 index 0000000..f391c32 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/BlockingIteratorIterator.java @@ -0,0 +1,24 @@ +package org.lucares.performance.db; + +import java.util.Iterator; +import java.util.Optional; + +final class BlockingIteratorIterator implements BlockingIterator { + + private final Iterator iterator; + + public BlockingIteratorIterator(final Iterator iterator) { + this.iterator = iterator; + } + + @Override + public Optional next() throws InterruptedException { + + if (iterator.hasNext()) { + final E next = iterator.next(); + return Optional.of(next); + } else { + return Optional.empty(); + } + } +} \ No newline at end of file diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java new file mode 100644 index 0000000..c882d9f --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/BlockingQueueIterator.java @@ -0,0 +1,35 @@ +package org.lucares.performance.db; + +import java.util.Optional; +import java.util.concurrent.BlockingQueue; + +final class BlockingQueueIterator implements BlockingIterator { + + private final BlockingQueue queue; + + private boolean closed = false; + + private final E poison; + + public BlockingQueueIterator(final BlockingQueue queue, final E poison) { + this.queue = queue; + this.poison = poison; + } + + @Override + public Optional next() throws InterruptedException { + + if (closed) { + return Optional.empty(); + } + + final E next = queue.take(); + + if (next == poison) { + closed = true; + return Optional.empty(); + } + + return Optional.of(next); + } +} \ No newline at end of file diff --git a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java index f1c221f..ebfe249 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java @@ -1,6 +1,9 @@ package org.lucares.performance.db; import java.text.SimpleDateFormat; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.Calendar; import java.util.Date; import java.util.TimeZone; @@ -8,70 +11,26 @@ import java.util.TimeZone; public class DateUtils { private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC"); - private static final SimpleDateFormat YEAR = new SimpleDateFormat("yyyy"); - private static final SimpleDateFormat MONTH = new SimpleDateFormat("MM"); - private static final SimpleDateFormat DAY = new SimpleDateFormat("dd"); - public static final synchronized Day getDay(final Date date) { + public static long getDateOffset(final OffsetDateTime date) { - final String yearString = YEAR.format(date); - final String monthString = MONTH.format(date); - final String dayString = DAY.format(date); - - final int year = Integer.parseInt(yearString, 10); - final int month = Integer.parseInt(monthString, 10); - final int day = Integer.parseInt(dayString, 10); - - return new Day(year, month, day); - } - - public static long getDateOffset(final Date date) { - - return getMidnightSameDay(date).getTime(); + return date.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli(); } public static Calendar getCalendar() { return Calendar.getInstance(TIME_ZONE_UTC); } - public static Date getMidnightSameDay(final Date date) { + public static OffsetDateTime getLastInstantOfDay(final OffsetDateTime date) { - final Calendar exactTime = getCalendar(); - exactTime.setTime(date); - - final Calendar midnight = getCalendar(); - final int year = exactTime.get(Calendar.YEAR); - final int month = exactTime.get(Calendar.MONTH); - final int day = exactTime.get(Calendar.DATE); - midnight.set(year, month, day); - midnight.set(Calendar.HOUR, 0); - midnight.set(Calendar.MINUTE, 0); - midnight.set(Calendar.SECOND, 0); - midnight.set(Calendar.MILLISECOND, 0); - - return midnight.getTime(); + return date.truncatedTo(ChronoUnit.DAYS).plusDays(1).minusNanos(1); } - public static Date getMidnightNextDay(final Date date) { - final Calendar exactTime = Calendar.getInstance(TIME_ZONE_UTC); - exactTime.setTime(date); + public static OffsetDateTime getDate(final int year, final int month, final int day, final int hour, + final int minute, final int second) { - exactTime.add(Calendar.DATE, 1); - exactTime.add(Calendar.MILLISECOND, -1); - - return exactTime.getTime(); - } - - public static Date getDate(final int year, final int month, final int day, final int hour, final int minute, - final int second) { - - final Calendar calendar = getCalendar(); - calendar.set(year, month - 1, day); - calendar.set(Calendar.HOUR, hour); - calendar.set(Calendar.MINUTE, minute); - calendar.set(Calendar.SECOND, second); - calendar.set(Calendar.MILLISECOND, 0); - return calendar.getTime(); + final OffsetDateTime result = OffsetDateTime.of(year, month, day, hour, minute, second, 0, ZoneOffset.UTC); + return result; } public static String format(final Date date) { @@ -80,4 +39,8 @@ public class DateUtils { return dateFormat.format(date); } + + public static OffsetDateTime nowInUtc() { + return OffsetDateTime.now(ZoneOffset.UTC); + } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Day.java b/performanceDb/src/main/java/org/lucares/performance/db/Day.java index 1a499f9..0e12d22 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Day.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Day.java @@ -1,27 +1,44 @@ package org.lucares.performance.db; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQuery; + class Day { - private final int year; - private final int month; - private final int day; + + private final OffsetDateTime date; public Day(final int year, final int month, final int day) { - super(); - this.year = year; - this.month = month; - this.day = day; + date = OffsetDateTime.of(year, month, day, 0, 0, 0, 0, ZoneOffset.UTC); + } + + public Day(final OffsetDateTime date) { + this.date = date.truncatedTo(ChronoUnit.DAYS); + } + + public Day() { + date = OffsetDateTime.now(ZoneOffset.UTC); } public int getYear() { - return year; + return date.getYear(); } public int getMonth() { - return month; + return date.getMonthValue(); } public int getDay() { - return day; + return date.getDayOfMonth(); + } + + public long getOffsetInEpochMilli() { + return date.toInstant().toEpochMilli(); } @Override @@ -30,7 +47,67 @@ class Day { } public String format(final String separator) { - return String.format("%04d%s%02d%s%02d", year, separator, month, separator, day); + + final String pattern = createPattern(separator); + final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + return date.format(formatter); + } + + private static String createPattern(final String separator) { + return String.format("yyyy%1$sMM%1$sdd", separator); + } + + public String serialize() { + + return format("-"); + } + + public static Day fromString(final String dateOffset) { + + final String pattern = createPattern("-"); + final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); + final TemporalQuery query = new TemporalQuery() { + + @Override + public OffsetDateTime queryFrom(final TemporalAccessor temporal) { + final LocalDate localDate = LocalDate.from(temporal); + final LocalTime localTime = LocalTime.MIDNIGHT; + return OffsetDateTime.of(localDate, localTime, ZoneOffset.UTC); + } + }; + final OffsetDateTime date = formatter.parse(dateOffset, query); + return new Day(date); + } + + public TimeRange toTimeRange() { + final OffsetDateTime from = date; + final OffsetDateTime to = DateUtils.getLastInstantOfDay(from); + return new TimeRange(from, to); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((date == null) ? 0 : date.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Day other = (Day) obj; + if (date == null) { + if (other.date != null) + return false; + } else if (!date.equals(other.date)) + return false; + return true; } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Entry.java b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java index 82ba180..dd7e611 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Entry.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java @@ -1,36 +1,54 @@ package org.lucares.performance.db; -import java.util.Date; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Comparator; public class Entry { - private final Date date; + public static final Comparator BY_DATE = new EntryByDateComparator(); + + private final long epochMilli; private final long value; - public Entry(final Date date, final long value) { + public Entry(final OffsetDateTime date, final long value) { super(); - this.date = date; + this.epochMilli = date.toInstant().toEpochMilli(); this.value = value; } - public Date getDate() { - return date; + Entry(final long epochMilli, final long value) { + super(); + this.epochMilli = epochMilli; + this.value = value; + } + + public OffsetDateTime getDate() { + final Instant instant = Instant.ofEpochMilli(epochMilli); + return OffsetDateTime.ofInstant(instant, ZoneOffset.UTC); } public long getValue() { return value; } + long getEpochMilli() { + return epochMilli; + } + @Override public String toString() { - return DateUtils.format(date) + " = " + value; + final OffsetDateTime date = getDate(); + return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value; } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((date == null) ? 0 : date.hashCode()); + result = prime * result + (int) (epochMilli ^ (epochMilli >>> 32)); result = prime * result + (int) (value ^ (value >>> 32)); return result; } @@ -44,13 +62,11 @@ public class Entry { if (getClass() != obj.getClass()) return false; final Entry other = (Entry) obj; - if (date == null) { - if (other.date != null) - return false; - } else if (!date.equals(other.date)) + if (epochMilli != other.epochMilli) return false; if (value != other.value) return false; return true; } + } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/EntryByDateComparator.java b/performanceDb/src/main/java/org/lucares/performance/db/EntryByDateComparator.java new file mode 100644 index 0000000..94dd837 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/EntryByDateComparator.java @@ -0,0 +1,19 @@ +package org.lucares.performance.db; + +import java.util.Comparator; + +public class EntryByDateComparator implements Comparator { + + @Override + public int compare(final Entry o1, final Entry o2) { + + long result = o1.getEpochMilli() - o2.getEpochMilli(); + + if (result == 0) { + result = o1.getValue() - o2.getValue(); + } + + return result < 0 ? -1 : (result == 0 ? 0 : 1); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java index 10a7985..010ff44 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java @@ -1,35 +1,23 @@ package org.lucares.performance.db; import java.io.File; -import java.util.Date; class PdbFile { private final Tags tags; - private final long dateOffset; + private final Day day; private final File file; - public PdbFile(final long dateOffset, final File file, final Tags tags) { - checkOffset(dateOffset); - this.dateOffset = dateOffset; + public PdbFile(final Day day, final File file, final Tags tags) { + this.day = day; this.file = file; this.tags = tags; } public static PdbFile today(final File file, final Tags tags) { - final long dateOffset = DateUtils.getDateOffset(new Date()); - return new PdbFile(dateOffset, file, tags); - } - - private void checkOffset(final long dateOffset) { - - final Date date = new Date(dateOffset); - final long expectedDateOffset = DateUtils.getDateOffset(date); - if (dateOffset != expectedDateOffset) { - throw new IllegalArgumentException("dateOffset must be at exactly midnight UTC: " + dateOffset + " != " - + expectedDateOffset + "(" + date + " != " + new Date(expectedDateOffset) + ")"); - } + final Day day = new Day(); + return new PdbFile(day, file, tags); } public Tags getTags() { @@ -40,14 +28,17 @@ class PdbFile { return file; } - public long getDateOffset() { - return dateOffset; + public Day getDay() { + return day; } public TimeRange getTimeRange() { - final Date from = new Date(dateOffset); - final Date to = DateUtils.getMidnightNextDay(from); - return new TimeRange(from, to); + + return day.toTimeRange(); + } + + public long getOffsetInEpochMilli() { + return getTimeRange().getFrom().toInstant().toEpochMilli(); } @Override @@ -59,7 +50,7 @@ class PdbFile { public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + (int) (dateOffset ^ (dateOffset >>> 32)); + result = prime * result + ((day == null) ? 0 : day.hashCode()); result = prime * result + ((file == null) ? 0 : file.hashCode()); result = prime * result + ((tags == null) ? 0 : tags.hashCode()); return result; @@ -74,7 +65,10 @@ class PdbFile { if (getClass() != obj.getClass()) return false; final PdbFile other = (PdbFile) obj; - if (dateOffset != other.dateOffset) + if (day == null) { + if (other.day != null) + return false; + } else if (!day.equals(other.day)) return false; if (file == null) { if (other.file != null) @@ -88,4 +82,5 @@ class PdbFile { return false; return true; } + } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java index 26ff5c4..ac66b0f 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java @@ -1,5 +1,6 @@ package org.lucares.performance.db; +import java.time.OffsetDateTime; import java.util.Comparator; public class PdbFileByTimeAsc implements Comparator { @@ -7,8 +8,9 @@ public class PdbFileByTimeAsc implements Comparator { @Override public int compare(final PdbFile o1, final PdbFile o2) { - final long difference = o1.getDateOffset() - o2.getDateOffset(); - return difference < 0 ? -1 : (difference == 0 ? 0 : 1); + final OffsetDateTime o1From = o1.getTimeRange().getFrom(); + final OffsetDateTime o2From = o2.getTimeRange().getFrom(); + return o1From.compareTo(o2From); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index 8d54057..82fc093 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -3,7 +3,9 @@ package org.lucares.performance.db; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; -import java.util.Date; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Optional; class PdbReader implements AutoCloseable { @@ -36,18 +38,26 @@ class PdbReader implements AutoCloseable { /** * Reads the next date value. * - * @return the date, or {@code null} if end of stream has been reached + * @return the date, or {@code -1} if end of stream has been reached * @throws IOException */ - public Date readDate() { + public long readEpochMilli() { assertPositionIsADatePosition(); final long value = read(); if (value < 0) { + return -1; + } + return pdbFile.getDay().getOffsetInEpochMilli() + value; + } + + public OffsetDateTime readDate() { + final long epochMilli = readEpochMilli(); + + if (epochMilli < 0) { return null; } - return new Date(pdbFile.getDateOffset() + value); - + return Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); } // visible for test @@ -174,8 +184,8 @@ class PdbReader implements AutoCloseable { } public Optional readEntry() throws ReadRuntimeException { - final Date date = readDate(); - if (date == null) { + final long epochMilli = readEpochMilli(); + if (epochMilli < 0) { return Optional.empty(); } final long value = readValue(); @@ -183,7 +193,7 @@ class PdbReader implements AutoCloseable { if (value < 0) { return Optional.empty(); } - return Optional.of(new Entry(date, value)); + return Optional.of(new Entry(epochMilli, value)); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java index 055d551..d292bc1 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -5,7 +5,6 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.Date; class PdbWriter implements AutoCloseable { @@ -24,12 +23,12 @@ class PdbWriter implements AutoCloseable { } public void write(final Entry entry) throws WriteException { - write(entry.getDate(), entry.getValue()); + write(entry.getEpochMilli(), entry.getValue()); } - void write(final Date time, final long value) throws WriteException { - final long timeValue = time.getTime(); - final long adjustedValue = timeValue - pdbFile.getDateOffset(); + void write(final long epochMilli, final long value) throws WriteException { + final long offsetEpochMill = pdbFile.getOffsetInEpochMilli(); + final long adjustedValue = epochMilli - offsetEpochMill; assertValueInRange(adjustedValue); assertValueInRange(value); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index 845fa9f..f284ef6 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -2,12 +2,14 @@ package org.lucares.performance.db; import java.io.IOException; import java.nio.file.Path; +import java.time.OffsetDateTime; import java.util.Arrays; -import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Spliterator; import java.util.Spliterators; +import java.util.concurrent.BlockingQueue; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,7 +26,7 @@ public class PerformanceDb implements AutoCloseable { tagsToFile = new TagsToFile(dataDirectory); } - public void put(final Date date, final long value, final Tags tags) throws WriteException { + public void put(final OffsetDateTime date, final long value, final Tags tags) throws WriteException { put(new Entry(date, value), tags); } @@ -33,31 +35,53 @@ public class PerformanceDb implements AutoCloseable { } public void put(final Iterable entries, final Tags tags) throws WriteException { + put(entries.iterator(), tags); + } + + public void put(final BlockingQueue entries, final Entry poisonObject, final Tags tags) + throws WriteException { + final BlockingQueueIterator iterator = new BlockingQueueIterator<>(entries, poisonObject); + put(iterator, tags); + } + + public void put(final Iterator entries, final Tags tags) throws WriteException { + + final BlockingIteratorIterator iterator = new BlockingIteratorIterator<>(entries); + put(iterator, tags); + } + + public void put(final BlockingIterator entries, final Tags tags) throws WriteException { final long start = System.nanoTime(); - double timeSpendInWrite = 0.0; + final double timeSpendInWrite = 0.0; long count = 0; PdbWriter writer = null; PdbFile pdbFile = null; try { - for (final Entry entry : entries) { - final Date date = entry.getDate(); + while (true) { + final Optional entryOptional = entries.next(); + if (!entryOptional.isPresent()) { + break; + } + final Entry entry = entryOptional.get(); + + final long epochMilli = entry.getEpochMilli(); final long value = entry.getValue(); if (pdbFile == null // - || !pdbFile.getTimeRange().inRange(date)) // TODO @ahr - // correct - // would be - // to check - // if the - // date is - // in the - // available - // range + || !pdbFile.getTimeRange().inRange(epochMilli)) // TODO + // @ahr + // correct + // would be + // to check + // if the + // date is + // in the + // available + // range { - final long startWrite = System.nanoTime(); + final OffsetDateTime date = entry.getDate(); pdbFile = tagsToFile.getFile(date, tags); - timeSpendInWrite += (System.nanoTime() - startWrite) / 1_000_000.0; } if (writer == null || !writer.getFile().equals(pdbFile)) { @@ -67,10 +91,13 @@ public class PerformanceDb implements AutoCloseable { writer = new PdbWriter(pdbFile); } - writer.write(date, value); + writer.write(epochMilli, value); count++; } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.info("Thread was interrupted. Aborting exectution."); } catch (final IOException e) { throw new WriteException(e); } finally { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index 9edf53f..b1a151e 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -5,9 +5,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.UUID; @@ -31,7 +31,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { try { db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); try { - db.createField(Fields.DATE_OFFSET, FieldType.BIGINT); + db.createField(Fields.DATE_OFFSET, FieldType.STRING); } catch (final FieldExistsException e) { // TODO @ahr ludb needs a hasField method, or a // createIfNotExists @@ -69,9 +69,10 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { private PdbFile toPdbFile(final Document document) { final File file = document.getFile(); - final long dateOffset = document.getPropertyLong(Fields.DATE_OFFSET); + final String dateOffset = document.getPropertyString(Fields.DATE_OFFSET); + final Day day = Day.fromString(dateOffset); final Tags tagsOfFile = toTags(document); - final PdbFile pdbFile = new PdbFile(dateOffset, file, tagsOfFile); + final PdbFile pdbFile = new PdbFile(day, file, tagsOfFile); return pdbFile; } @@ -88,7 +89,7 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { return tagsOfFile; } - public PdbFile getFile(final Date date, final Tags tags) throws FileNotFoundException, IOException { + public PdbFile getFile(final OffsetDateTime date, final Tags tags) throws FileNotFoundException, IOException { final List pdbFiles = getFilesMatchingTagsExactly(tags); final List preResult = new ArrayList<>(); @@ -123,21 +124,20 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { try (PdbReader reader = new PdbReader(pdbFile)) { if (reader.canSeekTail(2)) { reader.seekTail(2); - final Date lastDate = reader.readDate(); + final OffsetDateTime lastWrittenDate = reader.readDate(); - return new TimeRange(lastDate, pdbFile.getTimeRange().getTo()); + return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo()); } else { return pdbFile.getTimeRange(); } } } - private PdbFile createNewPdbFile(final Date date, final Tags tags) { + private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags) { final File file; - final long dateOffset; PdbFile result; file = createNewFile(date, tags); - dateOffset = DateUtils.getDateOffset(date); + final Day day = new Day(date); db.addDocument(file); @@ -147,9 +147,9 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { db.setProperty(file, Fields.prefixedKey(key), value); }); - db.setProperty(file, Fields.DATE_OFFSET, dateOffset); + db.setProperty(file, Fields.DATE_OFFSET, day.serialize()); - result = new PdbFile(dateOffset, file, tags); + result = new PdbFile(day, file, tags); return result; } @@ -168,8 +168,8 @@ public class TagsToFile implements AutoCloseable, CollectionUtils { }); } - private File createNewFile(final Date date, final Tags tags) { - final Day day = DateUtils.getDay(date); + private File createNewFile(final OffsetDateTime date, final Tags tags) { + final Day day = new Day(date); final String name = tags.abbreviatedRepresentation() + UUID.randomUUID().toString(); final File result = StorageUtils.createStorageFile(dataDirectory, day, name); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java b/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java index 2ccd036..42968f7 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java @@ -1,14 +1,17 @@ package org.lucares.performance.db; -import java.util.Date; -import java.util.concurrent.TimeUnit; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; public class TimeRange { - private final Date from; - private final Date to; + private final OffsetDateTime from; + private final OffsetDateTime to; - public TimeRange(final Date from, final Date to) { - if (!from.before(to)) { + public TimeRange(final OffsetDateTime from, final OffsetDateTime to) { + if (from.isAfter(to)) { throw new IllegalArgumentException("from date must be before to date. from: " + from + " to: " + to); } @@ -16,20 +19,27 @@ public class TimeRange { this.to = to; } - public Date getFrom() { + public OffsetDateTime getFrom() { return from; } - public Date getTo() { + public OffsetDateTime getTo() { return to; } - public long length(final TimeUnit timeUnit) { - final long duration = to.getTime() - from.getTime(); - return timeUnit.convert(duration, TimeUnit.MILLISECONDS); + public Duration duration() { + return Duration.between(from, to); } - public boolean inRange(final Date date) { + public boolean inRange(final long epochMilli) { + // TODO @ahr cache the epoch millis + final long fromEpochMilli = from.toInstant().toEpochMilli(); + final long toEpochMilli = to.toInstant().toEpochMilli(); + + return fromEpochMilli <= epochMilli && epochMilli <= toEpochMilli; + } + + public boolean inRange(final OffsetDateTime date) { return from.compareTo(date) <= 0 && to.compareTo(date) >= 0; } @@ -42,14 +52,17 @@ public class TimeRange { @Override public String toString() { - return "[" + from + ":" + to + "]"; + + final DateTimeFormatter formatter = DateTimeFormatter.ISO_ZONED_DATE_TIME.withZone(ZoneOffset.UTC); + final String fromUtc = from.format(formatter); + final String totc = from.format(formatter); + + return "[" + fromUtc + ":" + totc + "]"; } - public static TimeRange today() { - - final Date now = new Date(); - final Date from = DateUtils.getMidnightSameDay(now); - final Date to = DateUtils.getMidnightNextDay(now); + public static TimeRange ofDay(final OffsetDateTime day) { + final OffsetDateTime from = day.truncatedTo(ChronoUnit.DAYS); + final OffsetDateTime to = from.plusDays(1).minusNanos(1); return new TimeRange(from, to); } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java index 17820e2..dcc80ab 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java @@ -4,7 +4,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Date; +import java.time.OffsetDateTime; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -41,7 +41,8 @@ public class PdbReaderWriterTest { final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); final PdbFile pdbFile = PdbFile.today(file, new Tags()); - final Date now = new Date(); // TODO @ahr might fail at midnight + final OffsetDateTime now = OffsetDateTime.now(); // TODO @ahr might fail + // at midnight final Entry entry = new Entry(now, value); try (PdbWriter writer = new PdbWriter(pdbFile)) { diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java index e3f33b0..70759ae 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -4,14 +4,15 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -35,7 +36,7 @@ public class PerformanceDbTest { public void testInsertRead() throws Exception { try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { - final Date date = new Date(); + final OffsetDateTime date = DateUtils.nowInUtc(); final long value = 1; final Tags tags = new Tags("myKey", "myValue"); performanceDb.put(date, value, tags); @@ -51,8 +52,8 @@ public class PerformanceDbTest { public void testInsertIntoMultipleFilesRead() throws Exception { try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { - final Date dayOne = DateUtils.getDate(2016, 11, 1, 10, 0, 0); - final Date dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56); + final OffsetDateTime dayOne = DateUtils.getDate(2016, 11, 1, 10, 0, 0); + final OffsetDateTime dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56); final long valueOne = 1; final long valueTwo = 2; final Tags tags = new Tags("myKey", "myValue"); @@ -69,15 +70,14 @@ public class PerformanceDbTest { } } - private List generateEntries(final TimeRange timeRange, final long n) { + private List generateEntries(final TimeRange timeRange, final long n, final int addToDate) { final List result = new ArrayList<>(); - final long differenceInMs = timeRange.length(TimeUnit.MILLISECONDS) / n; - long currentTime = timeRange.getFrom().getTime(); + final long differenceInMs = timeRange.duration().toMillis() / n; + long currentTime = timeRange.getFrom().toInstant().toEpochMilli(); for (long i = 0; i < n; i++) { - final Date date = new Date(currentTime); final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE); - result.add(new Entry(date, value)); + result.add(new Entry(currentTime + addToDate, value)); currentTime += differenceInMs; } @@ -88,11 +88,11 @@ public class PerformanceDbTest { try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { - final TimeRange timeRange = TimeRange.today(); + final TimeRange timeRange = TimeRange.ofDay(OffsetDateTime.now(ZoneOffset.UTC)); final long numberOfEntries = 2; final Tags tags = new Tags("myKey", "one"); - final List entries = generateEntries(timeRange, numberOfEntries); + final List entries = generateEntries(timeRange, numberOfEntries, 0); printEntries(entries, ""); @@ -103,8 +103,8 @@ public class PerformanceDbTest { final List actualEntries = performanceDb.getAsList(tags); Assert.assertEquals(actualEntries, entries); - final File storageFileForToday = StorageUtils.createStorageFile(dataDirectory, - DateUtils.getDay(timeRange.getFrom()), "name doesn't matter"); + final File storageFileForToday = StorageUtils.createStorageFile(dataDirectory, new Day(timeRange.getFrom()), + "name doesn't matter"); final File storageFolderForToday = storageFileForToday.getParentFile(); final File[] filesInStorage = storageFolderForToday.listFiles(); Assert.assertEquals(filesInStorage.length, 1, @@ -115,27 +115,25 @@ public class PerformanceDbTest { public void testInsertIntoMultipleFilesWithDifferentTags() throws Exception { try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { - final Date from = DateUtils.getDate(2016, 1, 1, 00, 00, 00); - final Date to = DateUtils.getDate(2016, 12, 31, 23, 59, 59); + final OffsetDateTime from = DateUtils.getDate(2016, 1, 1, 00, 00, 00); + final OffsetDateTime to = DateUtils.getDate(2016, 1, 1, 23, 59, 50); final TimeRange timeRange = new TimeRange(from, to); - final long numberOfEntries = timeRange.length(TimeUnit.DAYS) * 2; // two - // entries - // per - // day + final long numberOfEntries = timeRange.duration().toHours(); + final Tags tagsCommon = new Tags("commonKey", "commonValue"); final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue"); - final List entriesOne = generateEntries(timeRange, numberOfEntries); + final List entriesOne = generateEntries(timeRange, numberOfEntries, 1); printEntries(entriesOne, "one"); performanceDb.put(entriesOne, tagsOne); final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue"); - final List entriesTwo = generateEntries(timeRange, numberOfEntries); + final List entriesTwo = generateEntries(timeRange, numberOfEntries, 2); printEntries(entriesTwo, "two"); performanceDb.put(entriesTwo, tagsTwo); final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue"); - final List entriesThree = generateEntries(timeRange, numberOfEntries); + final List entriesThree = generateEntries(timeRange, numberOfEntries, 3); printEntries(entriesThree, "three"); performanceDb.put(entriesThree, tagsThree); @@ -148,6 +146,13 @@ public class PerformanceDbTest { final List actualEntriesThree = performanceDb.getAsList(tagsThree); Assert.assertEquals(actualEntriesThree, entriesThree); + final List actualEntriesAll = performanceDb.getAsList(tagsCommon); + final List expectedAll = CollectionUtils.collate(entriesOne, + CollectionUtils.collate(entriesTwo, entriesThree, Entry.BY_DATE), Entry.BY_DATE); + + actualEntriesAll.sort(Entry.BY_DATE); + Assert.assertEquals(actualEntriesAll, expectedAll); + } } @@ -155,8 +160,12 @@ public class PerformanceDbTest { int index = 0; for (final Entry entry : entriesOne) { - System.out.printf("%4d %s %d (%s)\n", index, DateUtils.format(entry.getDate()), entry.getValue(), label); + System.out.printf("%4d %s %d (%s)\n", index, entry.getDate(), entry.getValue(), label); index++; } } + + public void testBlockingIteratorInput() throws Exception { + + } } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java index 0bf7f71..a991668 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -3,7 +3,8 @@ package org.lucares.performance.db; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Date; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -28,7 +29,7 @@ public class TagsToFilesTest { public void test() throws Exception { try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) { - final Date date = new Date(); + final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); final Tags tags = new Tags("myKey", "myValue"); final PdbFile newFileForTags = tagsToFile.getFile(date, tags); diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java index c252499..a7129d4 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java @@ -1,7 +1,9 @@ package org.lucares.performance.db; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Date; import java.util.List; import org.testng.Assert; @@ -15,10 +17,10 @@ public class TimeRangeTest { Object[][] providerIntersect() { final List result = new ArrayList<>(); - final Date a = new Date(1000); - final Date b = new Date(2000); - final Date c = new Date(3000); - final Date d = new Date(4000); + final OffsetDateTime a = Instant.ofEpochMilli(1000).atOffset(ZoneOffset.UTC); + final OffsetDateTime b = Instant.ofEpochMilli(2000).atOffset(ZoneOffset.UTC); + final OffsetDateTime c = Instant.ofEpochMilli(3000).atOffset(ZoneOffset.UTC); + final OffsetDateTime d = Instant.ofEpochMilli(4000).atOffset(ZoneOffset.UTC); result.add(new Object[] { new TimeRange(a, b), new TimeRange(c, d), false }); result.add(new Object[] { new TimeRange(a, c), new TimeRange(b, d), true }); @@ -34,4 +36,5 @@ public class TimeRangeTest { Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b); Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b); } + } diff --git a/recommind-logs/.gitignore b/recommind-logs/.gitignore new file mode 100644 index 0000000..267b0fd --- /dev/null +++ b/recommind-logs/.gitignore @@ -0,0 +1,5 @@ +/bin/ +/test-output/ +/.settings/ +/.classpath +/.project diff --git a/recommind-logs/build.gradle b/recommind-logs/build.gradle new file mode 100644 index 0000000..d7a74dd --- /dev/null +++ b/recommind-logs/build.gradle @@ -0,0 +1,6 @@ + +dependencies { + compile project(':performanceDb') + compile "io.thekraken:grok:0.1.5" +} + diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/LogReader.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/LogReader.java new file mode 100644 index 0000000..ca8403d --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/LogReader.java @@ -0,0 +1,37 @@ +package org.lucares.recommind.logs; + +import java.util.Iterator; + +import org.lucares.performance.db.Entry; + +import io.thekraken.grok.api.Grok; +import io.thekraken.grok.api.Match; + +public class LogReader implements Iterable { + + private final Grok grok; + + public LogReader(final Grok grok) { + super(); + this.grok = grok; + } + + @Override + public Iterator iterator() { + + // Grok grok = Grok.create("patterns/patterns"); + + /** Grok pattern to compile, here httpd logs */ + // grok.compile("%{COMBINEDAPACHELOG}"); + + /** Line of log to match */ + final String log = "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET / HTTP/1.1\" 200 44346 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""; + + final Match gm = grok.match(log); + gm.captures(); + gm.toMap(); + + return null; + } + +} diff --git a/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java b/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java new file mode 100644 index 0000000..f37946f --- /dev/null +++ b/recommind-logs/src/main/java/org/lucares/recommind/logs/PerformanceLogs.java @@ -0,0 +1,19 @@ +package org.lucares.recommind.logs; + +import java.io.File; +import java.util.concurrent.ArrayBlockingQueue; + +import org.lucares.performance.db.Entry; +import org.lucares.performance.db.PerformanceDb; +import org.lucares.performance.db.Tags; + +public class PerformanceLogs { + + public void ingest(final PerformanceDb db, final File performanceLog, final Tags tags) { + + final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(10); + + db.put(queue, tags); + + } +}