diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java b/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java deleted file mode 100644 index 2ed0aef..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.lucares.performance.db; - -class BitFiddling { - - static byte long3(final long x) { - return (byte) (x >> 24); - } - - static byte long2(final long x) { - return (byte) (x >> 16); - } - - static byte long1(final long x) { - return (byte) (x >> 8); - } - - static byte long0(final long x) { - return (byte) (x); - } - - static long makeLong(final byte b3, final byte b2, final byte b1, final byte b0) { - return ((((long) b3 & 0xff) << 24) | // - (((long) b2 & 0xff) << 16) | // - (((long) b1 & 0xff) << 8) | // - (((long) b0 & 0xff)));// - } -} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java index 7d3d45d..c1461f5 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java @@ -1,9 +1,11 @@ package org.lucares.performance.db; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -22,7 +24,19 @@ public interface CollectionUtils { return result; } - public default List filter(final Collection list, final Predicate predicate) { - return list.stream().filter(predicate).collect(Collectors.toList()); + public default List filter(final Collection collection, final Predicate predicate) { + return collection.stream().filter(predicate).collect(Collectors.toList()); + } + + public default List map(final Collection collection, final Function mapper) { + return collection.stream().map(mapper).collect(Collectors.toList()); + } + + public default List sorted(final Collection collection, final Comparator comparator) { + return collection.stream().sorted(comparator).collect(Collectors.toList()); + } + + public default Optional findFirst(final Collection collection) { + return collection.stream().findFirst(); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java index d829269..168e6dc 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java @@ -1,34 +1,11 @@ package org.lucares.performance.db; -import java.time.LocalDate; -import java.time.LocalTime; +import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAccessor; -import java.time.temporal.TemporalQuery; -import java.util.Calendar; -import java.util.TimeZone; public class DateUtils { - private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC"); - - public static long getDateOffset(final OffsetDateTime date) { - - return date.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli(); - } - - public static Calendar getCalendar() { - return Calendar.getInstance(TIME_ZONE_UTC); - } - - public static OffsetDateTime getLastInstantOfDay(final OffsetDateTime date) { - - return date.truncatedTo(ChronoUnit.DAYS).plusDays(1).minusNanos(1); - } - public static OffsetDateTime getDate(final int year, final int month, final int day, final int hour, final int minute, final int second) { @@ -40,18 +17,7 @@ public class DateUtils { return OffsetDateTime.now(ZoneOffset.UTC); } - public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter, - final ZoneOffset zoneOffset) { - final TemporalQuery query = new TemporalQuery() { - - @Override - public OffsetDateTime queryFrom(final TemporalAccessor temporal) { - final LocalDate localDate = LocalDate.from(temporal); - final LocalTime localTime = LocalTime.from(temporal); - return OffsetDateTime.of(localDate, localTime, zoneOffset); - } - }; - final OffsetDateTime date = formatter.parse(text, query); - return date; + public static OffsetDateTime epochMilliInUTC(final long lastEpochMilli) { + return OffsetDateTime.ofInstant(Instant.ofEpochMilli(lastEpochMilli), ZoneOffset.UTC); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Day.java b/performanceDb/src/main/java/org/lucares/performance/db/Day.java deleted file mode 100644 index 0e12d22..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/Day.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.lucares.performance.db; - -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.time.temporal.TemporalAccessor; -import java.time.temporal.TemporalQuery; - -class Day { - - private final OffsetDateTime date; - - public Day(final int year, final int month, final int day) { - date = OffsetDateTime.of(year, month, day, 0, 0, 0, 0, ZoneOffset.UTC); - } - - public Day(final OffsetDateTime date) { - this.date = date.truncatedTo(ChronoUnit.DAYS); - } - - public Day() { - date = OffsetDateTime.now(ZoneOffset.UTC); - } - - public int getYear() { - return date.getYear(); - } - - public int getMonth() { - return date.getMonthValue(); - } - - public int getDay() { - return date.getDayOfMonth(); - } - - public long getOffsetInEpochMilli() { - return date.toInstant().toEpochMilli(); - } - - @Override - public String toString() { - return format("-"); - } - - public String format(final String separator) { - - final String pattern = createPattern(separator); - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); - return date.format(formatter); - } - - private static String createPattern(final String separator) { - return String.format("yyyy%1$sMM%1$sdd", separator); - } - - public String serialize() { - - return format("-"); - } - - public static Day fromString(final String dateOffset) { - - final String pattern = createPattern("-"); - final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); - final TemporalQuery query = new TemporalQuery() { - - @Override - public OffsetDateTime queryFrom(final TemporalAccessor temporal) { - final LocalDate localDate = LocalDate.from(temporal); - final LocalTime localTime = LocalTime.MIDNIGHT; - return OffsetDateTime.of(localDate, localTime, ZoneOffset.UTC); - } - }; - final OffsetDateTime date = formatter.parse(dateOffset, query); - return new Day(date); - } - - public TimeRange toTimeRange() { - final OffsetDateTime from = date; - final OffsetDateTime to = DateUtils.getLastInstantOfDay(from); - return new TimeRange(from, to); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((date == null) ? 0 : date.hashCode()); - return result; - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final Day other = (Day) obj; - if (date == null) { - if (other.date != null) - return false; - } else if (!date.equals(other.date)) - return false; - return true; - } - -} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java b/performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java deleted file mode 100644 index 783833b..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.lucares.performance.db; - -import java.io.IOException; -import java.time.OffsetDateTime; - -import org.lucares.pdb.api.Tags; -import org.lucares.performance.db.PdbWriterManager.PdbFileSupplier; - -class FileSupplier implements PdbFileSupplier { - - private final TagsToFile tagsToFile; - - public FileSupplier(final TagsToFile tagsToFile) { - super(); - this.tagsToFile = tagsToFile; - } - - @Override - public PdbFile supply(final Tags tags, final OffsetDateTime date) { - try { - final PdbFile pdbFile = tagsToFile.getFile(date, tags); - return pdbFile; - } catch (final IOException e) { - throw new RuntimeException(e); - } - } -} \ No newline at end of file diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Group.java b/performanceDb/src/main/java/org/lucares/performance/db/Group.java index 7bb1f01..e4f5b1c 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Group.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Group.java @@ -7,19 +7,23 @@ import org.lucares.pdb.api.Tags; class Group { private final Tags tags; - private final List files; + private final List readers; - public Group(final Tags tags, final List files) { + public Group(final Tags tags, final List files) { super(); this.tags = tags; - this.files = files; + this.readers = files; } public Tags getTags() { return tags; } - public List getFiles() { - return files; + public List getReaders() { + return readers; + } + + public void addReader(final PdbReader pdbReader) { + readers.add(pdbReader); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Grouping.java b/performanceDb/src/main/java/org/lucares/performance/db/Grouping.java index 15659a3..506f6dc 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/Grouping.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/Grouping.java @@ -21,23 +21,23 @@ public class Grouping { this.groups.addAll(groups); } - public static Grouping groupBy(final List pdbFiles, final String groupByField) { + public static Grouping groupBy(final List pdbReaders, final String groupByField) { final Grouping result; if (groupByField == NO_GROUPING) { - final Group group = new Group(null, pdbFiles); + final Group group = new Group(null, pdbReaders); result = new Grouping(group); } else { final Map grouping = new HashMap<>(); - for (final PdbFile pdbFile : pdbFiles) { - final Tags tags = pdbFile.getTags(); + for (final PdbReader pdbReader : pdbReaders) { + final Tags tags = pdbReader.getPdbFile().getTags(); final String value = tags.getValue(groupByField); if (value != null) { addIfNotExists(grouping, groupByField, value); - grouping.get(value).getFiles().add(pdbFile); + grouping.get(value).addReader(pdbReader); } } result = new Grouping(grouping.values()); @@ -49,9 +49,9 @@ public class Grouping { final String value) { if (!grouping.containsKey(value)) { final Tags tags = Tags.create(groupByField, value); - final List files = new ArrayList<>(); + final List readers = new ArrayList<>(); - grouping.put(value, new Group(tags, files)); + grouping.put(value, new Group(tags, readers)); } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java index cc320e9..cb242d8 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java @@ -1,6 +1,8 @@ package org.lucares.performance.db; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import org.lucares.pdb.api.Tags; @@ -69,4 +71,12 @@ class PdbFile { return false; return true; } + + public boolean exists() throws ReadException { + try { + return Files.isRegularFile(path) && Files.size(path) >= ByteType.VersionByte.MIN_LENGTH; + } catch (final IOException e) { + throw new ReadException(e); + } + } } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java deleted file mode 100644 index 57fa754..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.lucares.performance.db; - -import java.time.OffsetDateTime; -import java.util.Comparator; - -public class PdbFileByTimeAsc implements Comparator { - - public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc(); - - @Override - public int compare(final PdbFileOffsetTime o1, final PdbFileOffsetTime o2) { - - final OffsetDateTime o1From = o1.getOffsetTime(); - final OffsetDateTime o2From = o2.getOffsetTime(); - return o1From.compareTo(o2From); - } - -} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java index ce16921..46a47a6 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -1,31 +1,23 @@ package org.lucares.performance.db; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Files; import java.util.ArrayDeque; import java.util.Collection; import java.util.Iterator; import java.util.Optional; import java.util.Queue; import java.util.function.Supplier; -import java.util.logging.Level; -import java.util.logging.Logger; import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Tags; public class PdbFileIterator implements Iterator, AutoCloseable { - private final static Logger LOGGER = Logger.getLogger(PdbFileIterator.class.getCanonicalName()); - private static final class EntrySupplier implements Supplier, AutoCloseable { - private final Queue pdbFiles; + private final Queue pdbFiles; private PdbReader reader; - private PdbFile currentPdbFile; - public EntrySupplier(final Collection pdbFiles) { + public EntrySupplier(final Collection pdbFiles) { super(); this.pdbFiles = new ArrayDeque<>(pdbFiles); } @@ -39,14 +31,14 @@ public class PdbFileIterator implements Iterator, AutoCloseable { if (reader == null) { return null; } - final Entry entry = reader.readNullableEntry(currentPdbFile.getTags()); + final Entry entry = reader.readNullableEntry(reader.getPdbFile().getTags()); if (entry == null) { nextFile(); if (reader == null) { return null; } else { - final Tags tags = currentPdbFile.getTags(); + final Tags tags = reader.getPdbFile().getTags(); return reader.readEntry(tags).orElse(null); } } @@ -62,22 +54,7 @@ public class PdbFileIterator implements Iterator, AutoCloseable { reader = null; } - while (!pdbFiles.isEmpty()) { - currentPdbFile = pdbFiles.poll(); - try { - - if (Files.size(currentPdbFile.getPath()) > 0) { - reader = new PdbReader(currentPdbFile); - break; - } else { - LOGGER.info("ignoring empty file " + currentPdbFile); - } - } catch (final FileNotFoundException e) { - LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e); - } catch (final IOException e) { - throw new ReadException(e); - } - } + reader = pdbFiles.poll(); } @Override @@ -85,6 +62,14 @@ public class PdbFileIterator implements Iterator, AutoCloseable { if (reader != null) { reader.close(); } + + while (!pdbFiles.isEmpty()) { + try { + pdbFiles.poll().close(); + } catch (final Exception e) { + e.printStackTrace(); + } + } } } @@ -93,7 +78,7 @@ public class PdbFileIterator implements Iterator, AutoCloseable { private Optional next = Optional.empty(); - public PdbFileIterator(final Collection pdbFiles) { + public PdbFileIterator(final Collection pdbFiles) { supplier = new EntrySupplier(pdbFiles); } @@ -117,7 +102,7 @@ public class PdbFileIterator implements Iterator, AutoCloseable { } @Override - public void close() throws Exception { + public void close() { supplier.close(); } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java index ab83fb9..037cddc 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -25,13 +25,21 @@ class PdbReader implements AutoCloseable { private long index = 0; private int peekedByte = PEEK_NOT_SET; - public PdbReader(final PdbFile pdbFile) throws FileNotFoundException { + private final PdbFile pdbFile; + + public PdbReader(final PdbFile pdbFile) throws ReadException { super(); - final File storageFile = pdbFile.getPath().toFile(); + try { + this.pdbFile = pdbFile; + final File storageFile = pdbFile.getPath().toFile(); - this.data = new BufferedInputStream(new FileInputStream(storageFile)); + this.data = new BufferedInputStream(new FileInputStream(storageFile)); + + init(); + } catch (final FileNotFoundException e) { + throw new ReadException(e); + } - init(); } private void init() { @@ -49,6 +57,10 @@ class PdbReader implements AutoCloseable { } } + public PdbFile getPdbFile() { + return pdbFile; + } + /** * Seek to the end of the file. * @@ -161,5 +173,4 @@ class PdbReader implements AutoCloseable { final long bytePrefix = ByteType.CONTINUATION.getBytePrefix(); return bytePrefix == (nextByte & bytePrefix); } - } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java index 35a4163..c7df71f 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -3,10 +3,10 @@ package org.lucares.performance.db; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.Flushable; import java.io.IOException; import java.io.OutputStream; import java.time.OffsetDateTime; -import java.util.logging.Logger; import org.lucares.pdb.api.Entry; @@ -68,9 +68,7 @@ import org.lucares.pdb.api.Entry; * 1 and has room for 7 bits. The result looks like this: 01000001 * 11001010 */ -class PdbWriter implements AutoCloseable { - - private final static Logger LOGGER = Logger.getLogger(PdbWriter.class.getCanonicalName()); +class PdbWriter implements AutoCloseable, Flushable { private static final boolean APPEND = true; @@ -100,10 +98,14 @@ class PdbWriter implements AutoCloseable { } } - public PdbFile getFile() { + public PdbFile getPdbFile() { return pdbFile; } + public OffsetDateTime getDateOffset() { + return DateUtils.epochMilliInUTC(lastEpochMilli); + } + public void write(final Entry entry) throws WriteException { System.out.println(entry); final long epochMilli = entry.getEpochMilli(); @@ -137,6 +139,7 @@ class PdbWriter implements AutoCloseable { outputStream.close(); } + @Override public void flush() throws IOException { outputStream.flush(); } @@ -172,4 +175,5 @@ class PdbWriter implements AutoCloseable { public static void init(final PdbFile result) throws IOException { writeEntry(result); } + } diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterByTimeAsc.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterByTimeAsc.java new file mode 100644 index 0000000..3028f0d --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterByTimeAsc.java @@ -0,0 +1,18 @@ +package org.lucares.performance.db; + +import java.time.OffsetDateTime; +import java.util.Comparator; + +public class PdbWriterByTimeAsc implements Comparator { + + public static final PdbWriterByTimeAsc INSTANCE = new PdbWriterByTimeAsc(); + + @Override + public int compare(final PdbWriter o1, final PdbWriter o2) { + + final OffsetDateTime o1From = o1.getDateOffset(); + final OffsetDateTime o2From = o2.getDateOffset(); + return o1From.compareTo(o2From); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java deleted file mode 100644 index 30ce947..0000000 --- a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.lucares.performance.db; - -import java.io.IOException; -import java.time.OffsetDateTime; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.lucares.pdb.api.Tags; - -public class PdbWriterManager implements AutoCloseable { - - private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName()); - - public interface PdbFileSupplier { - public PdbFile supply(Tags tags, OffsetDateTime date); - } - - final Map map = new HashMap<>(); - - private final PdbFileSupplier supplier; - - private Day lastDay = new Day(OffsetDateTime.MIN); - - public PdbWriterManager(final PdbFileSupplier supplier) { - this.supplier = supplier; - } - - public PdbWriter getWriter(final Tags tags, final OffsetDateTime date) throws IOException { - - handleDateChange(date); - - final PdbFile pdbFile = supplier.supply(tags, date); - - if (!map.containsKey(pdbFile)) { - final PdbWriter writer = new PdbWriter(pdbFile); - map.put(pdbFile, writer); - } - return map.get(pdbFile); - } - - private void handleDateChange(final OffsetDateTime date) { - - final Day day = new Day(date); - - if (!day.equals(lastDay)) { - LOGGER.info("finished with " + day); - closeFiles(); - lastDay = day; - } - } - - public void flush() { - LOGGER.info("flushing all files"); - for (final PdbWriter writer : map.values()) { - try { - writer.flush(); - } catch (final IOException e) { - LOGGER.log(Level.WARNING, e.getMessage(), e); - } - } - } - - @Override - public void close() { - closeFiles(); - } - - private void closeFiles() { - final Iterator> it = map.entrySet().iterator(); - - while (it.hasNext()) { - final Entry entry = it.next(); - final PdbWriter writer = entry.getValue(); - try { - writer.close(); - - } catch (final IOException e) { - LOGGER.log(Level.WARNING, e.getMessage(), e); - } - it.remove(); - } - } -} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index c540d0d..4e0a1dc 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -1,7 +1,6 @@ package org.lucares.performance.db; import java.io.File; -import java.io.IOException; import java.nio.file.Path; import java.time.OffsetDateTime; import java.util.ArrayList; @@ -61,7 +60,7 @@ public class PerformanceDb implements AutoCloseable { long count = 0; double durationInManager = 0; - try (final PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) { + try { long start = System.nanoTime(); while (true) { @@ -75,7 +74,7 @@ public class PerformanceDb implements AutoCloseable { final Tags tags = entry.getTags(); final OffsetDateTime date = entry.getDate(); - final PdbWriter writer = manager.getWriter(tags, date); + final PdbWriter writer = tagsToFile.getWriter(date, tags); writer.write(entry); count++; @@ -94,13 +93,13 @@ public class PerformanceDb implements AutoCloseable { } } - } catch (final IOException e) { + } catch (final RuntimeException e) { throw new WriteException(e); } catch (final InterruptedException e) { Thread.currentThread().interrupt(); LOGGER.info("Thread was interrupted. Aborting exectution."); } finally { - // + tagsToFile.flush(); } } @@ -123,9 +122,9 @@ public class PerformanceDb implements AutoCloseable { */ public Result get(final String query, final String groupBy) { - final List pdbFiles = tagsToFile.getFilesForQuery(query); + final List pdbReaders = tagsToFile.getReaders(query); - final Grouping grouping = Grouping.groupBy(pdbFiles, groupBy); + final Grouping grouping = Grouping.groupBy(pdbReaders, groupBy); final Result result = toResult(grouping); @@ -135,7 +134,7 @@ public class PerformanceDb implements AutoCloseable { private Result toResult(final Grouping grouping) { final List groupResults = new ArrayList<>(); for (final Group group : grouping.getGroups()) { - final Stream stream = toStream(group.getFiles()); + final Stream stream = toStream(group.getReaders()); final GroupResult groupResult = new GroupResult(stream, group.getTags()); groupResults.add(groupResult); } @@ -143,11 +142,19 @@ public class PerformanceDb implements AutoCloseable { return result; } - private Stream toStream(final List pdbFiles) { - final Iterator iterator = new PdbFileIterator(pdbFiles); + private Stream toStream(final List pdbFiles) { + final PdbFileIterator iterator = new PdbFileIterator(pdbFiles); final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); - return StreamSupport.stream(spliterator, false); + final Stream stream = StreamSupport.stream(spliterator, false); + final Stream result = stream.onClose(() -> { + try { + iterator.close(); + } catch (final RuntimeException e) { + e.printStackTrace(); + } + }); + return result; } @Override @@ -158,6 +165,8 @@ public class PerformanceDb implements AutoCloseable { // H2 doesn't actually do anything in close throw new IllegalStateException(e); } + + tagsToFile.close(); } public List autocomplete(final String query, final int caretIndex) { diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java index 3f086fa..1243713 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -1,21 +1,30 @@ package org.lucares.performance.db; -import java.io.FileNotFoundException; +import java.io.Flushable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import org.lucares.ludb.Document; import org.lucares.ludb.H2DB; import org.lucares.pdb.api.Tags; -public class TagsToFile implements CollectionUtils { +public class TagsToFile implements CollectionUtils, AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(TagsToFile.class.getCanonicalName()); private static class TagSpecificBaseDir { private final Path path; @@ -37,9 +46,36 @@ public class TagsToFile implements CollectionUtils { } } + private static class WriterCache { + final List writers = new ArrayList<>(); + + public List getWriters() { + return writers; + } + + public void addWriter(final PdbWriter writer) { + writers.add(writer); + } + + public Optional writer(final PdbFile pdbFile) { + return writers.stream().filter(w -> Objects.equals(w.getPdbFile(), pdbFile)).findAny(); + } + + public Optional tagSpecificBaseDir() { + + if (writers.size() > 0) { + return Optional.of(writers.get(0).getPdbFile().getPath().getParent()); + } + return Optional.empty(); + } + + } + private final H2DB db; private final Path dataDirectory; + private final Map cachedWriters = new HashMap<>(); + public TagsToFile(final Path dataDirectory, final H2DB db) { this.dataDirectory = dataDirectory; this.db = db; @@ -57,7 +93,20 @@ public class TagsToFile implements CollectionUtils { return getFilesForQuery(query); } - List getFilesForQuery(final String query) { + public List getReaders(final String query) { + final List result = new ArrayList<>(); + + final List filesForQuery = getFilesForQuery(query); + + for (final PdbFile pdbFile : filesForQuery) { + final PdbReader reader = new PdbReader(pdbFile); + result.add(reader); + } + + return result; + } + + public List getFilesForQuery(final String query) { final List result = new ArrayList<>(); final List tagSpecificFolders = getTagSpecificFolders(query); @@ -107,46 +156,55 @@ public class TagsToFile implements CollectionUtils { return tagsOfFile; } - public PdbFile getFile(final OffsetDateTime date, final Tags tags) throws FileNotFoundException, IOException { + public PdbWriter getWriter(final OffsetDateTime date, final Tags tags) throws ReadException, WriteException { final List pdbFiles = getFilesMatchingTagsExactly(tags); - final List preResult = new ArrayList<>(); assertAllFilesHaveSameFolder(pdbFiles); - PdbFile result; - for (final PdbFile pdbFile : pdbFiles) { + final WriterCache writersForTags = getOrInit(tags); - if (Files.isRegularFile(pdbFile.getPath()) - && Files.size(pdbFile.getPath()) >= ByteType.VersionByte.MIN_LENGTH) { + pdbFiles.removeIf(f -> !f.exists()); + final List> optionalWriters = map(pdbFiles, writersForTags::writer); + final List> existingWriters = filter(optionalWriters, Optional::isPresent); + final List writers = map(existingWriters, Optional::get); + final List candidateWriters = filter(writers, writer -> { + final OffsetDateTime offsetTime = writer.getDateOffset(); + return !date.isBefore(offsetTime); + }); + final List sortedCanditateWriters = sorted(candidateWriters, PdbWriterByTimeAsc.INSTANCE.reversed()); + final Optional optionalFirst = findFirst(sortedCanditateWriters); - final OffsetDateTime offsetTime = PdbFileUtils.dateOffset(pdbFile); - - if (!offsetTime.isAfter(date)) { - preResult.add(new PdbFileOffsetTime(pdbFile, offsetTime)); - } - } - } - - if (preResult.isEmpty()) { - - Path tagSpecificStorageFolder; - if (pdbFiles.isEmpty()) { - tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags); - } else { - final Path storageFilePath = pdbFiles.get(0).getPath(); - tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath); - } - - result = createNewPdbFile(tags, tagSpecificStorageFolder); - } else { - Collections.sort(preResult, PdbFileByTimeAsc.INSTANCE.reversed()); - result = preResult.get(0).getPdbFile(); - } + final PdbWriter result = optionalFirst.orElseGet(() -> newPdbWriter(tags)); return result; } + private WriterCache getOrInit(final Tags tags) { + + if (!cachedWriters.containsKey(tags)) { + cachedWriters.put(tags, new WriterCache()); + } + + return cachedWriters.get(tags); + } + + private PdbWriter newPdbWriter(final Tags tags) { + try { + PdbWriter result; + final Path tagSpecificStorageFolder = getOrInit(tags).tagSpecificBaseDir() + .orElse(StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags)); + + final PdbFile pdbFile = createNewPdbFile(tags, tagSpecificStorageFolder); + result = new PdbWriter(pdbFile); + + getOrInit(tags).addWriter(result); + return result; + } catch (final IOException e) { + throw new WriteException(e); + } + } + private void assertAllFilesHaveSameFolder(final List pdbFiles) { final Set reducedFolder = pdbFiles.stream()// .map(PdbFile::getPath)// @@ -190,4 +248,40 @@ public class TagsToFile implements CollectionUtils { return result; } + + @SuppressWarnings("unchecked") + private void forEachWriter(final Consumer consumer) { + for (final Entry readersWriters : cachedWriters.entrySet()) { + + for (final PdbWriter writer : readersWriters.getValue().getWriters()) { + try { + consumer.accept((T) writer); + } catch (final RuntimeException e) { + LOGGER.log(Level.WARNING, "failed to close writer for file " + writer.getPdbFile().getPath(), e); + } + } + } + } + + @Override + public void close() { + + forEachWriter(t -> { + try { + t.close(); + } catch (final Exception e) { + throw new WriteException(e); + } + }); + } + + public void flush() { + forEachWriter(t -> { + try { + t.flush(); + } catch (final IOException e) { + throw new WriteException(e); + } + }); + } } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java b/performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java deleted file mode 100644 index 4a40546..0000000 --- a/performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.lucares.performance.db; - -import org.testng.annotations.Test; - -@Test -public class BitFiddlingTest { - - // TODO @ahr remove or move - // public void testEncodingMeasurement() throws Exception { - // - // final List types = Arrays.asList(ByteType.DATE_INCREMENT, - // ByteType.DATE_OFFSET, ByteType.MEASUREMENT, - // ByteType.VERSION); - // - // final List values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L, - // 202L, 255L, 256L, 8191L, 8192L, 1048575L, - // 1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L, - // 2199023255552L, 281474976710655L, - // 281474976710656L, 36028797018963967L, 36028797018963968L, - // 4611686018427387901L, 4611686018427387904L); - // - // for (final Long value : values) { - // for (final ByteType type : types) { - // encodeDecode(value, type); - // } - // } - // } - // - // private void encodeDecode(final Long value, final ByteType byteType) - // throws Exception { - // - // final ByteArrayOutputStream output = new ByteArrayOutputStream(); - // BitFiddling.writeValue(value, byteType, output); - // - // final byte[] byteArray = output.toByteArray(); - // final ByteArrayInputStream input = new ByteArrayInputStream(byteArray); - // final Long readValue = BitFiddling.readValue(byteType, input); - // - // Assert.assertEquals(readValue, value); - // } - -} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java deleted file mode 100644 index 23cb98b..0000000 --- a/performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.lucares.performance.db; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; - -import org.lucares.ludb.H2DB; -import org.lucares.pdb.api.Entry; -import org.lucares.pdb.api.Tags; -import org.testng.Assert; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -@Test -public class PdbWriterManagerTest { - - private Path dataDirectory; - - @BeforeMethod - public void beforeMethod() throws IOException { - dataDirectory = Files.createTempDirectory("pdb"); - } - - @AfterMethod - public void afterMethod() throws IOException { - FileUtils.delete(dataDirectory); - } - - @Test - public void testManager() throws Exception { - try (H2DB db = new H2DB(dataDirectory.toFile())) { - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); - - final Tags tagsA = Tags.create("key", "A"); - final Tags tagsB = Tags.create("key", "B"); - - try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) { - - final OffsetDateTime date = OffsetDateTime.now(); - - final PdbWriter firstWriterForTagsA = manager.getWriter(tagsA, date); - final PdbWriter secondWriterForTagsA = manager.getWriter(tagsA, date); - final PdbWriter firstWriterForTagsB = manager.getWriter(tagsB, date); - - Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA); - Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB); - } - } - } - - @Test - public void testManager2() throws Exception { - - try (H2DB db = new H2DB(dataDirectory.toFile())) { - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); - - final Tags tags = Tags.create("key", "A"); - - try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) { - - final OffsetDateTime morning = OffsetDateTime.of(2016, 1, 1, 8, 0, 0, 0, ZoneOffset.UTC); - final OffsetDateTime noon = OffsetDateTime.of(2016, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC); - final OffsetDateTime afternoon = OffsetDateTime.of(2016, 1, 1, 17, 0, 0, 0, ZoneOffset.UTC); - - final PdbWriter writerNoon = manager.getWriter(tags, noon); - writerNoon.write(new Entry(noon, 1, tags)); - - final PdbWriter writerMorning = manager.getWriter(tags, morning); - writerMorning.write(new Entry(morning, 2, tags)); - - final PdbWriter writerAfternoon = manager.getWriter(tags, afternoon); - writerAfternoon.write(new Entry(afternoon, 3, tags)); - - Assert.assertSame(writerNoon, writerAfternoon); - Assert.assertNotSame(writerNoon, writerMorning); - } - } - } -} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java index aba752e..8fb24dd 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -163,7 +163,6 @@ public class PerformanceDbTest { actualEntriesAll.sort(EntryByDateComparator.INSTANCE); Assert.assertEquals(actualEntriesAll, expectedAll); - } } diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java index a5b0aaa..717e244 100644 --- a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -32,49 +32,44 @@ public class TagsToFilesTest { public void test() throws Exception { - try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); + try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); + final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) { final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); final Tags tags = Tags.create("myKey", "myValue"); - final PdbFile newFileForTags = tagsToFile.getFile(date, tags); - PdbWriter.writeEntry(newFileForTags); + final PdbWriter newFileForTags = tagsToFile.getWriter(date, tags); - final PdbFile existingFileForTags = tagsToFile.getFile(date, tags); + final PdbWriter existingFileForTags = tagsToFile.getWriter(date, tags); - Assert.assertEquals(newFileForTags, existingFileForTags); + Assert.assertSame(newFileForTags, existingFileForTags); } } public void testAppendingToSameFileIfNewDateIsAfter() throws Exception { - try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); + try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); + final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) { final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1); final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1); final Tags tags = Tags.create("myKey", "myValue"); - final PdbFile fileForDay1 = tagsToFile.getFile(day1, tags); - final PdbFile fileForDay2 = tagsToFile.getFile(day2, tags); + final PdbWriter writerForDay1 = tagsToFile.getWriter(day1, tags); + writerForDay1.write(new Entry(day1, 1, tags)); + final PdbWriter writerForDay2 = tagsToFile.getWriter(day2, tags); + writerForDay2.write(new Entry(day2, 2, tags)); - Assert.assertEquals(fileForDay1, fileForDay2); - - final PdbFile existingFileForDay1 = tagsToFile.getFile(day1, tags); - Assert.assertEquals(fileForDay1, existingFileForDay1); + Assert.assertSame(writerForDay1, writerForDay2); } } @Test(invocationCount = 1) public void testNewFileIfDateIsTooOld() throws Exception { - try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); + try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); + final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) { final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1); @@ -83,43 +78,46 @@ public class TagsToFilesTest { final Tags tags = Tags.create("myKey", "myValue"); - final PdbFile fileAfternoon = tagsToFile.getFile(afternoon, tags); - PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags)); + final PdbWriter writerAfternoon = tagsToFile.getWriter(afternoon, tags); + writerAfternoon.write(new Entry(afternoon, 1, tags)); - final PdbFile fileMorning = tagsToFile.getFile(morning, tags); - PdbWriter.writeEntry(fileMorning, new Entry(morning, 2, tags)); + final PdbWriter writerMorning = tagsToFile.getWriter(morning, tags); + writerMorning.write(new Entry(morning, 2, tags)); - Assert.assertNotEquals(fileAfternoon, fileMorning); + Assert.assertNotSame(writerAfternoon, writerMorning); + Assert.assertNotEquals(writerAfternoon.getPdbFile(), writerMorning.getPdbFile()); - final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags); - PdbWriter.writeEntry(fileEarlyMorning, new Entry(earlyMorning, 3, tags)); + final PdbWriter writerEarlyMorning = tagsToFile.getWriter(earlyMorning, tags); + writerEarlyMorning.write(new Entry(earlyMorning, 3, tags)); - Assert.assertNotEquals(fileEarlyMorning, fileAfternoon); - Assert.assertNotEquals(fileEarlyMorning, fileMorning); + Assert.assertNotSame(writerEarlyMorning, writerAfternoon); + Assert.assertNotSame(writerEarlyMorning, writerMorning); - final PdbFile fileEvening = tagsToFile.getFile(evening, tags); + final PdbWriter writerEvening = tagsToFile.getWriter(evening, tags); - Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file"); - Assert.assertNotEquals(fileEvening, fileMorning); - Assert.assertNotEquals(fileEvening, fileEarlyMorning); + Assert.assertSame(writerEvening, writerAfternoon, + "the evening event can be appended to the afternoon file"); + Assert.assertNotSame(writerEvening, writerMorning); + Assert.assertNotSame(writerEvening, writerEarlyMorning); + Assert.assertNotEquals(writerEvening.getPdbFile(), writerMorning.getPdbFile()); + Assert.assertNotEquals(writerEvening.getPdbFile(), writerEarlyMorning.getPdbFile()); } } public void testIdenticalDatesGoIntoSameFile() throws Exception { - try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { - - final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db); + try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); + final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) { final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final Tags tags = Tags.create("myKey", "myValue"); - final PdbFile fileA = tagsToFile.getFile(timestamp, tags); - PdbWriter.writeEntry(fileA, new Entry(timestamp, 1, tags)); + final PdbWriter fileA = tagsToFile.getWriter(timestamp, tags); + fileA.write(new Entry(timestamp, 1, tags)); - final PdbFile fileB = tagsToFile.getFile(timestamp, tags); - PdbWriter.writeEntry(fileA, new Entry(timestamp, 2, tags)); + final PdbWriter fileB = tagsToFile.getWriter(timestamp, tags); + fileA.write(new Entry(timestamp, 2, tags)); Assert.assertEquals(fileA, fileB); }