andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

finalize refactoring

Browse Source
This commit is contained in:
Andreas Huber
2016-12-29 18:27:15 +01:00
parent 68ac1dd631
commit de241ceb6d
20 changed files with 287 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java
View File

@@ -1,27 +0,0 @@
package org.lucares.performance.db;
class BitFiddling {
static byte long3(final long x) {
return (byte) (x >> 24);
}
static byte long2(final long x) {
return (byte) (x >> 16);
}
static byte long1(final long x) {
return (byte) (x >> 8);
}
static byte long0(final long x) {
return (byte) (x);
}
static long makeLong(final byte b3, final byte b2, final byte b1, final byte b0) {
return ((((long) b3 & 0xff) << 24) | //
(((long) b2 & 0xff) << 16) | //
(((long) b1 & 0xff) << 8) | //
(((long) b0 & 0xff)));//
}
}

18
performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java
View File

@@ -1,9 +1,11 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.util.Collection; import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -22,7 +24,19 @@ public interface CollectionUtils {
return result; return result;
} }
public default <T> List<T> filter(final Collection<T> list, final Predicate<T> predicate) { public default <T> List<T> filter(final Collection<T> collection, final Predicate<T> predicate) {
return list.stream().filter(predicate).collect(Collectors.toList()); return collection.stream().filter(predicate).collect(Collectors.toList());
}
public default <T, R> List<R> map(final Collection<T> collection, final Function<T, R> mapper) {
return collection.stream().map(mapper).collect(Collectors.toList());
}
public default <T> List<T> sorted(final Collection<T> collection, final Comparator<T> comparator) {
return collection.stream().sorted(comparator).collect(Collectors.toList());
}
public default <T> Optional<T> findFirst(final Collection<T> collection) {
return collection.stream().findFirst();
} }
} }

40
performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java
View File

@@ -1,34 +1,11 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.time.LocalDate; import java.time.Instant;
import java.time.LocalTime;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.time.ZoneOffset; import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQuery;
import java.util.Calendar;
import java.util.TimeZone;
public class DateUtils { public class DateUtils {
private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC");
public static long getDateOffset(final OffsetDateTime date) {
return date.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli();
}
public static Calendar getCalendar() {
return Calendar.getInstance(TIME_ZONE_UTC);
}
public static OffsetDateTime getLastInstantOfDay(final OffsetDateTime date) {
return date.truncatedTo(ChronoUnit.DAYS).plusDays(1).minusNanos(1);
}
public static OffsetDateTime getDate(final int year, final int month, final int day, final int hour, public static OffsetDateTime getDate(final int year, final int month, final int day, final int hour,
final int minute, final int second) { final int minute, final int second) {
@@ -40,18 +17,7 @@ public class DateUtils {
return OffsetDateTime.now(ZoneOffset.UTC); return OffsetDateTime.now(ZoneOffset.UTC);
} }
public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter, public static OffsetDateTime epochMilliInUTC(final long lastEpochMilli) {
final ZoneOffset zoneOffset) { return OffsetDateTime.ofInstant(Instant.ofEpochMilli(lastEpochMilli), ZoneOffset.UTC);
final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
@Override
public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
final LocalDate localDate = LocalDate.from(temporal);
final LocalTime localTime = LocalTime.from(temporal);
return OffsetDateTime.of(localDate, localTime, zoneOffset);
}
};
final OffsetDateTime date = formatter.parse(text, query);
return date;
} }
} }

113
performanceDb/src/main/java/org/lucares/performance/db/Day.java
View File

@@ -1,113 +0,0 @@
package org.lucares.performance.db;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQuery;
class Day {
private final OffsetDateTime date;
public Day(final int year, final int month, final int day) {
date = OffsetDateTime.of(year, month, day, 0, 0, 0, 0, ZoneOffset.UTC);
}
public Day(final OffsetDateTime date) {
this.date = date.truncatedTo(ChronoUnit.DAYS);
}
public Day() {
date = OffsetDateTime.now(ZoneOffset.UTC);
}
public int getYear() {
return date.getYear();
}
public int getMonth() {
return date.getMonthValue();
}
public int getDay() {
return date.getDayOfMonth();
}
public long getOffsetInEpochMilli() {
return date.toInstant().toEpochMilli();
}
@Override
public String toString() {
return format("-");
}
public String format(final String separator) {
final String pattern = createPattern(separator);
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
return date.format(formatter);
}
private static String createPattern(final String separator) {
return String.format("yyyy%1$sMM%1$sdd", separator);
}
public String serialize() {
return format("-");
}
public static Day fromString(final String dateOffset) {
final String pattern = createPattern("-");
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
@Override
public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
final LocalDate localDate = LocalDate.from(temporal);
final LocalTime localTime = LocalTime.MIDNIGHT;
return OffsetDateTime.of(localDate, localTime, ZoneOffset.UTC);
}
};
final OffsetDateTime date = formatter.parse(dateOffset, query);
return new Day(date);
}
public TimeRange toTimeRange() {
final OffsetDateTime from = date;
final OffsetDateTime to = DateUtils.getLastInstantOfDay(from);
return new TimeRange(from, to);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((date == null) ? 0 : date.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Day other = (Day) obj;
if (date == null) {
if (other.date != null)
return false;
} else if (!date.equals(other.date))
return false;
return true;
}
}

27
performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java
View File

@@ -1,27 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.OffsetDateTime;
import org.lucares.pdb.api.Tags;
import org.lucares.performance.db.PdbWriterManager.PdbFileSupplier;
class FileSupplier implements PdbFileSupplier {
private final TagsToFile tagsToFile;
public FileSupplier(final TagsToFile tagsToFile) {
super();
this.tagsToFile = tagsToFile;
}
@Override
public PdbFile supply(final Tags tags, final OffsetDateTime date) {
try {
final PdbFile pdbFile = tagsToFile.getFile(date, tags);
return pdbFile;
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}

14
performanceDb/src/main/java/org/lucares/performance/db/Group.java
View File

@@ -7,19 +7,23 @@ import org.lucares.pdb.api.Tags;
class Group { class Group {
private final Tags tags; private final Tags tags;
private final List<PdbFile> files; private final List<PdbReader> readers;
public Group(final Tags tags, final List<PdbFile> files) { public Group(final Tags tags, final List<PdbReader> files) {
super(); super();
this.tags = tags; this.tags = tags;
this.files = files; this.readers = files;
} }
public Tags getTags() { public Tags getTags() {
return tags; return tags;
} }
public List<PdbFile> getFiles() { public List<PdbReader> getReaders() {
return files; return readers;
}
public void addReader(final PdbReader pdbReader) {
readers.add(pdbReader);
} }
} }

14
performanceDb/src/main/java/org/lucares/performance/db/Grouping.java
View File

@@ -21,23 +21,23 @@ public class Grouping {
this.groups.addAll(groups); this.groups.addAll(groups);
} }
public static Grouping groupBy(final List<PdbFile> pdbFiles, final String groupByField) { public static Grouping groupBy(final List<PdbReader> pdbReaders, final String groupByField) {
final Grouping result; final Grouping result;
if (groupByField == NO_GROUPING) { if (groupByField == NO_GROUPING) {
final Group group = new Group(null, pdbFiles); final Group group = new Group(null, pdbReaders);
result = new Grouping(group); result = new Grouping(group);
} else { } else {
final Map<String, Group> grouping = new HashMap<>(); final Map<String, Group> grouping = new HashMap<>();
for (final PdbFile pdbFile : pdbFiles) { for (final PdbReader pdbReader : pdbReaders) {
final Tags tags = pdbFile.getTags(); final Tags tags = pdbReader.getPdbFile().getTags();
final String value = tags.getValue(groupByField); final String value = tags.getValue(groupByField);
if (value != null) { if (value != null) {
addIfNotExists(grouping, groupByField, value); addIfNotExists(grouping, groupByField, value);
grouping.get(value).getFiles().add(pdbFile); grouping.get(value).addReader(pdbReader);
} }
} }
result = new Grouping(grouping.values()); result = new Grouping(grouping.values());
@@ -49,9 +49,9 @@ public class Grouping {
final String value) { final String value) {
if (!grouping.containsKey(value)) { if (!grouping.containsKey(value)) {
final Tags tags = Tags.create(groupByField, value); final Tags tags = Tags.create(groupByField, value);
final List<PdbFile> files = new ArrayList<>(); final List<PdbReader> readers = new ArrayList<>();
grouping.put(value, new Group(tags, files)); grouping.put(value, new Group(tags, readers));
} }
} }

10
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -1,6 +1,8 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
@@ -69,4 +71,12 @@ class PdbFile {
return false; return false;
return true; return true;
} }
public boolean exists() throws ReadException {
try {
return Files.isRegularFile(path) && Files.size(path) >= ByteType.VersionByte.MIN_LENGTH;
} catch (final IOException e) {
throw new ReadException(e);
}
}
} }

18
performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java
View File

@@ -1,18 +0,0 @@
package org.lucares.performance.db;
import java.time.OffsetDateTime;
import java.util.Comparator;
public class PdbFileByTimeAsc implements Comparator<PdbFileOffsetTime> {
public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc();
@Override
public int compare(final PdbFileOffsetTime o1, final PdbFileOffsetTime o2) {
final OffsetDateTime o1From = o1.getOffsetTime();
final OffsetDateTime o2From = o2.getOffsetTime();
return o1From.compareTo(o2From);
}
}

45
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -1,31 +1,23 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.Collection; import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import java.util.Optional; import java.util.Optional;
import java.util.Queue; import java.util.Queue;
import java.util.function.Supplier; import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable { public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbFileIterator.class.getCanonicalName());
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable { private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles; private final Queue<PdbReader> pdbFiles;
private PdbReader reader; private PdbReader reader;
private PdbFile currentPdbFile;
public EntrySupplier(final Collection<PdbFile> pdbFiles) { public EntrySupplier(final Collection<PdbReader> pdbFiles) {
super(); super();
this.pdbFiles = new ArrayDeque<>(pdbFiles); this.pdbFiles = new ArrayDeque<>(pdbFiles);
} }
@@ -39,14 +31,14 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader == null) { if (reader == null) {
return null; return null;
} }
final Entry entry = reader.readNullableEntry(currentPdbFile.getTags()); final Entry entry = reader.readNullableEntry(reader.getPdbFile().getTags());
if (entry == null) { if (entry == null) {
nextFile(); nextFile();
if (reader == null) { if (reader == null) {
return null; return null;
} else { } else {
final Tags tags = currentPdbFile.getTags(); final Tags tags = reader.getPdbFile().getTags();
return reader.readEntry(tags).orElse(null); return reader.readEntry(tags).orElse(null);
} }
} }
@@ -62,22 +54,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
reader = null; reader = null;
} }
while (!pdbFiles.isEmpty()) { reader = pdbFiles.poll();
currentPdbFile = pdbFiles.poll();
try {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
}
} }
@Override @Override
@@ -85,6 +62,14 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader != null) { if (reader != null) {
reader.close(); reader.close();
} }
while (!pdbFiles.isEmpty()) {
try {
pdbFiles.poll().close();
} catch (final Exception e) {
e.printStackTrace();
}
}
} }
} }
@@ -93,7 +78,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private Optional<Entry> next = Optional.empty(); private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) { public PdbFileIterator(final Collection<PdbReader> pdbFiles) {
supplier = new EntrySupplier(pdbFiles); supplier = new EntrySupplier(pdbFiles);
} }
@@ -117,7 +102,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
} }
@Override @Override
public void close() throws Exception { public void close() {
supplier.close(); supplier.close();
} }

21
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -25,13 +25,21 @@ class PdbReader implements AutoCloseable {
private long index = 0; private long index = 0;
private int peekedByte = PEEK_NOT_SET; private int peekedByte = PEEK_NOT_SET;
public PdbReader(final PdbFile pdbFile) throws FileNotFoundException { private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws ReadException {
super(); super();
final File storageFile = pdbFile.getPath().toFile(); try {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile)); this.data = new BufferedInputStream(new FileInputStream(storageFile));
init();
} catch (final FileNotFoundException e) {
throw new ReadException(e);
}
init();
} }
private void init() { private void init() {
@@ -49,6 +57,10 @@ class PdbReader implements AutoCloseable {
} }
} }
public PdbFile getPdbFile() {
return pdbFile;
}
/** /**
* Seek to the end of the file. * Seek to the end of the file.
* *
@@ -161,5 +173,4 @@ class PdbReader implements AutoCloseable {
final long bytePrefix = ByteType.CONTINUATION.getBytePrefix(); final long bytePrefix = ByteType.CONTINUATION.getBytePrefix();
return bytePrefix == (nextByte & bytePrefix); return bytePrefix == (nextByte & bytePrefix);
} }
} }

14
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -3,10 +3,10 @@ package org.lucares.performance.db;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.logging.Logger;
import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.Entry;
@@ -68,9 +68,7 @@ import org.lucares.pdb.api.Entry;
* 1 and has room for 7 bits. The result looks like this: <b>01</b><i>000001<i> * 1 and has room for 7 bits. The result looks like this: <b>01</b><i>000001<i>
* <b>1</b><i>1001010</i> * <b>1</b><i>1001010</i>
*/ */
class PdbWriter implements AutoCloseable { class PdbWriter implements AutoCloseable, Flushable {
private final static Logger LOGGER = Logger.getLogger(PdbWriter.class.getCanonicalName());
private static final boolean APPEND = true; private static final boolean APPEND = true;
@@ -100,10 +98,14 @@ class PdbWriter implements AutoCloseable {
} }
} }
public PdbFile getFile() { public PdbFile getPdbFile() {
return pdbFile; return pdbFile;
} }
public OffsetDateTime getDateOffset() {
return DateUtils.epochMilliInUTC(lastEpochMilli);
}
public void write(final Entry entry) throws WriteException { public void write(final Entry entry) throws WriteException {
System.out.println(entry); System.out.println(entry);
final long epochMilli = entry.getEpochMilli(); final long epochMilli = entry.getEpochMilli();
@@ -137,6 +139,7 @@ class PdbWriter implements AutoCloseable {
outputStream.close(); outputStream.close();
} }
@Override
public void flush() throws IOException { public void flush() throws IOException {
outputStream.flush(); outputStream.flush();
} }
@@ -172,4 +175,5 @@ class PdbWriter implements AutoCloseable {
public static void init(final PdbFile result) throws IOException { public static void init(final PdbFile result) throws IOException {
writeEntry(result); writeEntry(result);
} }
} }

18
performanceDb/src/main/java/org/lucares/performance/db/PdbWriterByTimeAsc.java Normal file
View File

@@ -0,0 +1,18 @@
package org.lucares.performance.db;
import java.time.OffsetDateTime;
import java.util.Comparator;
public class PdbWriterByTimeAsc implements Comparator<PdbWriter> {
public static final PdbWriterByTimeAsc INSTANCE = new PdbWriterByTimeAsc();
@Override
public int compare(final PdbWriter o1, final PdbWriter o2) {
final OffsetDateTime o1From = o1.getDateOffset();
final OffsetDateTime o2From = o2.getDateOffset();
return o1From.compareTo(o2From);
}
}

87
performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java
View File

@@ -1,87 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lucares.pdb.api.Tags;
public class PdbWriterManager implements AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName());
public interface PdbFileSupplier {
public PdbFile supply(Tags tags, OffsetDateTime date);
}
final Map<PdbFile, PdbWriter> map = new HashMap<>();
private final PdbFileSupplier supplier;
private Day lastDay = new Day(OffsetDateTime.MIN);
public PdbWriterManager(final PdbFileSupplier supplier) {
this.supplier = supplier;
}
public PdbWriter getWriter(final Tags tags, final OffsetDateTime date) throws IOException {
handleDateChange(date);
final PdbFile pdbFile = supplier.supply(tags, date);
if (!map.containsKey(pdbFile)) {
final PdbWriter writer = new PdbWriter(pdbFile);
map.put(pdbFile, writer);
}
return map.get(pdbFile);
}
private void handleDateChange(final OffsetDateTime date) {
final Day day = new Day(date);
if (!day.equals(lastDay)) {
LOGGER.info("finished with " + day);
closeFiles();
lastDay = day;
}
}
public void flush() {
LOGGER.info("flushing all files");
for (final PdbWriter writer : map.values()) {
try {
writer.flush();
} catch (final IOException e) {
LOGGER.log(Level.WARNING, e.getMessage(), e);
}
}
}
@Override
public void close() {
closeFiles();
}
private void closeFiles() {
final Iterator<Entry<PdbFile, PdbWriter>> it = map.entrySet().iterator();
while (it.hasNext()) {
final Entry<PdbFile, PdbWriter> entry = it.next();
final PdbWriter writer = entry.getValue();
try {
writer.close();
} catch (final IOException e) {
LOGGER.log(Level.WARNING, e.getMessage(), e);
}
it.remove();
}
}
}

31
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -1,7 +1,6 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
@@ -61,7 +60,7 @@ public class PerformanceDb implements AutoCloseable {
long count = 0; long count = 0;
double durationInManager = 0; double durationInManager = 0;
try (final PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) { try {
long start = System.nanoTime(); long start = System.nanoTime();
while (true) { while (true) {
@@ -75,7 +74,7 @@ public class PerformanceDb implements AutoCloseable {
final Tags tags = entry.getTags(); final Tags tags = entry.getTags();
final OffsetDateTime date = entry.getDate(); final OffsetDateTime date = entry.getDate();
final PdbWriter writer = manager.getWriter(tags, date); final PdbWriter writer = tagsToFile.getWriter(date, tags);
writer.write(entry); writer.write(entry);
count++; count++;
@@ -94,13 +93,13 @@ public class PerformanceDb implements AutoCloseable {
} }
} }
} catch (final IOException e) { } catch (final RuntimeException e) {
throw new WriteException(e); throw new WriteException(e);
} catch (final InterruptedException e) { } catch (final InterruptedException e) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
LOGGER.info("Thread was interrupted. Aborting exectution."); LOGGER.info("Thread was interrupted. Aborting exectution.");
} finally { } finally {
// tagsToFile.flush();
} }
} }
@@ -123,9 +122,9 @@ public class PerformanceDb implements AutoCloseable {
*/ */
public Result get(final String query, final String groupBy) { public Result get(final String query, final String groupBy) {
final List<PdbFile> pdbFiles = tagsToFile.getFilesForQuery(query); final List<PdbReader> pdbReaders = tagsToFile.getReaders(query);
final Grouping grouping = Grouping.groupBy(pdbFiles, groupBy); final Grouping grouping = Grouping.groupBy(pdbReaders, groupBy);
final Result result = toResult(grouping); final Result result = toResult(grouping);
@@ -135,7 +134,7 @@ public class PerformanceDb implements AutoCloseable {
private Result toResult(final Grouping grouping) { private Result toResult(final Grouping grouping) {
final List<GroupResult> groupResults = new ArrayList<>(); final List<GroupResult> groupResults = new ArrayList<>();
for (final Group group : grouping.getGroups()) { for (final Group group : grouping.getGroups()) {
final Stream<Entry> stream = toStream(group.getFiles()); final Stream<Entry> stream = toStream(group.getReaders());
final GroupResult groupResult = new GroupResult(stream, group.getTags()); final GroupResult groupResult = new GroupResult(stream, group.getTags());
groupResults.add(groupResult); groupResults.add(groupResult);
} }
@@ -143,11 +142,19 @@ public class PerformanceDb implements AutoCloseable {
return result; return result;
} }
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) { private Stream<Entry> toStream(final List<PdbReader> pdbFiles) {
final Iterator<Entry> iterator = new PdbFileIterator(pdbFiles); final PdbFileIterator iterator = new PdbFileIterator(pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
return StreamSupport.stream(spliterator, false); final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
final Stream<Entry> result = stream.onClose(() -> {
try {
iterator.close();
} catch (final RuntimeException e) {
e.printStackTrace();
}
});
return result;
} }
@Override @Override
@@ -158,6 +165,8 @@ public class PerformanceDb implements AutoCloseable {
// H2 doesn't actually do anything in close // H2 doesn't actually do anything in close
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
tagsToFile.close();
} }
public List<Proposal> autocomplete(final String query, final int caretIndex) { public List<Proposal> autocomplete(final String query, final int caretIndex) {

160
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,21 +1,30 @@
package org.lucares.performance.db; package org.lucares.performance.db;
import java.io.FileNotFoundException; import java.io.Flushable;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.lucares.ludb.Document; import org.lucares.ludb.Document;
import org.lucares.ludb.H2DB; import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
public class TagsToFile implements CollectionUtils { public class TagsToFile implements CollectionUtils, AutoCloseable {
private static final Logger LOGGER = Logger.getLogger(TagsToFile.class.getCanonicalName());
private static class TagSpecificBaseDir { private static class TagSpecificBaseDir {
private final Path path; private final Path path;
@@ -37,9 +46,36 @@ public class TagsToFile implements CollectionUtils {
} }
} }
private static class WriterCache {
final List<PdbWriter> writers = new ArrayList<>();
public List<PdbWriter> getWriters() {
return writers;
}
public void addWriter(final PdbWriter writer) {
writers.add(writer);
}
public Optional<PdbWriter> writer(final PdbFile pdbFile) {
return writers.stream().filter(w -> Objects.equals(w.getPdbFile(), pdbFile)).findAny();
}
public Optional<Path> tagSpecificBaseDir() {
if (writers.size() > 0) {
return Optional.of(writers.get(0).getPdbFile().getPath().getParent());
}
return Optional.empty();
}
}
private final H2DB db; private final H2DB db;
private final Path dataDirectory; private final Path dataDirectory;
private final Map<Tags, WriterCache> cachedWriters = new HashMap<>();
public TagsToFile(final Path dataDirectory, final H2DB db) { public TagsToFile(final Path dataDirectory, final H2DB db) {
this.dataDirectory = dataDirectory; this.dataDirectory = dataDirectory;
this.db = db; this.db = db;
@@ -57,7 +93,20 @@ public class TagsToFile implements CollectionUtils {
return getFilesForQuery(query); return getFilesForQuery(query);
} }
List<PdbFile> getFilesForQuery(final String query) { public List<PdbReader> getReaders(final String query) {
final List<PdbReader> result = new ArrayList<>();
final List<PdbFile> filesForQuery = getFilesForQuery(query);
for (final PdbFile pdbFile : filesForQuery) {
final PdbReader reader = new PdbReader(pdbFile);
result.add(reader);
}
return result;
}
public List<PdbFile> getFilesForQuery(final String query) {
final List<PdbFile> result = new ArrayList<>(); final List<PdbFile> result = new ArrayList<>();
final List<TagSpecificBaseDir> tagSpecificFolders = getTagSpecificFolders(query); final List<TagSpecificBaseDir> tagSpecificFolders = getTagSpecificFolders(query);
@@ -107,46 +156,55 @@ public class TagsToFile implements CollectionUtils {
return tagsOfFile; return tagsOfFile;
} }
public PdbFile getFile(final OffsetDateTime date, final Tags tags) throws FileNotFoundException, IOException { public PdbWriter getWriter(final OffsetDateTime date, final Tags tags) throws ReadException, WriteException {
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags); final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
final List<PdbFileOffsetTime> preResult = new ArrayList<>();
assertAllFilesHaveSameFolder(pdbFiles); assertAllFilesHaveSameFolder(pdbFiles);
PdbFile result; final WriterCache writersForTags = getOrInit(tags);
for (final PdbFile pdbFile : pdbFiles) {
if (Files.isRegularFile(pdbFile.getPath()) pdbFiles.removeIf(f -> !f.exists());
&& Files.size(pdbFile.getPath()) >= ByteType.VersionByte.MIN_LENGTH) { final List<Optional<PdbWriter>> optionalWriters = map(pdbFiles, writersForTags::writer);
final List<Optional<PdbWriter>> existingWriters = filter(optionalWriters, Optional::isPresent);
final List<PdbWriter> writers = map(existingWriters, Optional::get);
final List<PdbWriter> candidateWriters = filter(writers, writer -> {
final OffsetDateTime offsetTime = writer.getDateOffset();
return !date.isBefore(offsetTime);
});
final List<PdbWriter> sortedCanditateWriters = sorted(candidateWriters, PdbWriterByTimeAsc.INSTANCE.reversed());
final Optional<PdbWriter> optionalFirst = findFirst(sortedCanditateWriters);
final OffsetDateTime offsetTime = PdbFileUtils.dateOffset(pdbFile); final PdbWriter result = optionalFirst.orElseGet(() -> newPdbWriter(tags));
if (!offsetTime.isAfter(date)) {
preResult.add(new PdbFileOffsetTime(pdbFile, offsetTime));
}
}
}
if (preResult.isEmpty()) {
Path tagSpecificStorageFolder;
if (pdbFiles.isEmpty()) {
tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
} else {
final Path storageFilePath = pdbFiles.get(0).getPath();
tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
}
result = createNewPdbFile(tags, tagSpecificStorageFolder);
} else {
Collections.sort(preResult, PdbFileByTimeAsc.INSTANCE.reversed());
result = preResult.get(0).getPdbFile();
}
return result; return result;
} }
private WriterCache getOrInit(final Tags tags) {
if (!cachedWriters.containsKey(tags)) {
cachedWriters.put(tags, new WriterCache());
}
return cachedWriters.get(tags);
}
private PdbWriter newPdbWriter(final Tags tags) {
try {
PdbWriter result;
final Path tagSpecificStorageFolder = getOrInit(tags).tagSpecificBaseDir()
.orElse(StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags));
final PdbFile pdbFile = createNewPdbFile(tags, tagSpecificStorageFolder);
result = new PdbWriter(pdbFile);
getOrInit(tags).addWriter(result);
return result;
} catch (final IOException e) {
throw new WriteException(e);
}
}
private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) { private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
final Set<Path> reducedFolder = pdbFiles.stream()// final Set<Path> reducedFolder = pdbFiles.stream()//
.map(PdbFile::getPath)// .map(PdbFile::getPath)//
@@ -190,4 +248,40 @@ public class TagsToFile implements CollectionUtils {
return result; return result;
} }
@SuppressWarnings("unchecked")
private <T extends AutoCloseable & Flushable> void forEachWriter(final Consumer<T> consumer) {
for (final Entry<Tags, WriterCache> readersWriters : cachedWriters.entrySet()) {
for (final PdbWriter writer : readersWriters.getValue().getWriters()) {
try {
consumer.accept((T) writer);
} catch (final RuntimeException e) {
LOGGER.log(Level.WARNING, "failed to close writer for file " + writer.getPdbFile().getPath(), e);
}
}
}
}
@Override
public void close() {
forEachWriter(t -> {
try {
t.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final IOException e) {
throw new WriteException(e);
}
});
}
} }

42
performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java
View File

@@ -1,42 +0,0 @@
package org.lucares.performance.db;
import org.testng.annotations.Test;
@Test
public class BitFiddlingTest {
// TODO @ahr remove or move
// public void testEncodingMeasurement() throws Exception {
//
// final List<ByteType> types = Arrays.asList(ByteType.DATE_INCREMENT,
// ByteType.DATE_OFFSET, ByteType.MEASUREMENT,
// ByteType.VERSION);
//
// final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L,
// 202L, 255L, 256L, 8191L, 8192L, 1048575L,
// 1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L,
// 2199023255552L, 281474976710655L,
// 281474976710656L, 36028797018963967L, 36028797018963968L,
// 4611686018427387901L, 4611686018427387904L);
//
// for (final Long value : values) {
// for (final ByteType type : types) {
// encodeDecode(value, type);
// }
// }
// }
//
// private void encodeDecode(final Long value, final ByteType byteType)
// throws Exception {
//
// final ByteArrayOutputStream output = new ByteArrayOutputStream();
// BitFiddling.writeValue(value, byteType, output);
//
// final byte[] byteArray = output.toByteArray();
// final ByteArrayInputStream input = new ByteArrayInputStream(byteArray);
// final Long readValue = BitFiddling.readValue(byteType, input);
//
// Assert.assertEquals(readValue, value);
// }
}

82
performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java
View File

@@ -1,82 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class PdbWriterManagerTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@Test
public void testManager() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final Tags tagsA = Tags.create("key", "A");
final Tags tagsB = Tags.create("key", "B");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime date = OffsetDateTime.now();
final PdbWriter firstWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter secondWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter firstWriterForTagsB = manager.getWriter(tagsB, date);
Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
}
}
}
@Test
public void testManager2() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final Tags tags = Tags.create("key", "A");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime morning = OffsetDateTime.of(2016, 1, 1, 8, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime noon = OffsetDateTime.of(2016, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime afternoon = OffsetDateTime.of(2016, 1, 1, 17, 0, 0, 0, ZoneOffset.UTC);
final PdbWriter writerNoon = manager.getWriter(tags, noon);
writerNoon.write(new Entry(noon, 1, tags));
final PdbWriter writerMorning = manager.getWriter(tags, morning);
writerMorning.write(new Entry(morning, 2, tags));
final PdbWriter writerAfternoon = manager.getWriter(tags, afternoon);
writerAfternoon.write(new Entry(afternoon, 3, tags));
Assert.assertSame(writerNoon, writerAfternoon);
Assert.assertNotSame(writerNoon, writerMorning);
}
}
}
}

1
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -163,7 +163,6 @@ public class PerformanceDbTest {
actualEntriesAll.sort(EntryByDateComparator.INSTANCE); actualEntriesAll.sort(EntryByDateComparator.INSTANCE);
Assert.assertEquals(actualEntriesAll, expectedAll); Assert.assertEquals(actualEntriesAll, expectedAll);
} }
} }

76
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -32,49 +32,44 @@ public class TagsToFilesTest {
public void test() throws Exception { public void test() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC); final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue"); final Tags tags = Tags.create("myKey", "myValue");
final PdbFile newFileForTags = tagsToFile.getFile(date, tags); final PdbWriter newFileForTags = tagsToFile.getWriter(date, tags);
PdbWriter.writeEntry(newFileForTags);
final PdbFile existingFileForTags = tagsToFile.getFile(date, tags); final PdbWriter existingFileForTags = tagsToFile.getWriter(date, tags);
Assert.assertEquals(newFileForTags, existingFileForTags); Assert.assertSame(newFileForTags, existingFileForTags);
} }
} }
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception { public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1); final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1); final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
final Tags tags = Tags.create("myKey", "myValue"); final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileForDay1 = tagsToFile.getFile(day1, tags); final PdbWriter writerForDay1 = tagsToFile.getWriter(day1, tags);
final PdbFile fileForDay2 = tagsToFile.getFile(day2, tags); writerForDay1.write(new Entry(day1, 1, tags));
final PdbWriter writerForDay2 = tagsToFile.getWriter(day2, tags);
writerForDay2.write(new Entry(day2, 2, tags));
Assert.assertEquals(fileForDay1, fileForDay2); Assert.assertSame(writerForDay1, writerForDay2);
final PdbFile existingFileForDay1 = tagsToFile.getFile(day1, tags);
Assert.assertEquals(fileForDay1, existingFileForDay1);
} }
} }
@Test(invocationCount = 1) @Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception { public void testNewFileIfDateIsTooOld() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1); final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -83,43 +78,46 @@ public class TagsToFilesTest {
final Tags tags = Tags.create("myKey", "myValue"); final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileAfternoon = tagsToFile.getFile(afternoon, tags); final PdbWriter writerAfternoon = tagsToFile.getWriter(afternoon, tags);
PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags)); writerAfternoon.write(new Entry(afternoon, 1, tags));
final PdbFile fileMorning = tagsToFile.getFile(morning, tags); final PdbWriter writerMorning = tagsToFile.getWriter(morning, tags);
PdbWriter.writeEntry(fileMorning, new Entry(morning, 2, tags)); writerMorning.write(new Entry(morning, 2, tags));
Assert.assertNotEquals(fileAfternoon, fileMorning); Assert.assertNotSame(writerAfternoon, writerMorning);
Assert.assertNotEquals(writerAfternoon.getPdbFile(), writerMorning.getPdbFile());
final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags); final PdbWriter writerEarlyMorning = tagsToFile.getWriter(earlyMorning, tags);
PdbWriter.writeEntry(fileEarlyMorning, new Entry(earlyMorning, 3, tags)); writerEarlyMorning.write(new Entry(earlyMorning, 3, tags));
Assert.assertNotEquals(fileEarlyMorning, fileAfternoon); Assert.assertNotSame(writerEarlyMorning, writerAfternoon);
Assert.assertNotEquals(fileEarlyMorning, fileMorning); Assert.assertNotSame(writerEarlyMorning, writerMorning);
final PdbFile fileEvening = tagsToFile.getFile(evening, tags); final PdbWriter writerEvening = tagsToFile.getWriter(evening, tags);
Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file"); Assert.assertSame(writerEvening, writerAfternoon,
Assert.assertNotEquals(fileEvening, fileMorning); "the evening event can be appended to the afternoon file");
Assert.assertNotEquals(fileEvening, fileEarlyMorning); Assert.assertNotSame(writerEvening, writerMorning);
Assert.assertNotSame(writerEvening, writerEarlyMorning);
Assert.assertNotEquals(writerEvening.getPdbFile(), writerMorning.getPdbFile());
Assert.assertNotEquals(writerEvening.getPdbFile(), writerEarlyMorning.getPdbFile());
} }
} }
public void testIdenticalDatesGoIntoSameFile() throws Exception { public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) { try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1); final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final Tags tags = Tags.create("myKey", "myValue"); final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileA = tagsToFile.getFile(timestamp, tags); final PdbWriter fileA = tagsToFile.getWriter(timestamp, tags);
PdbWriter.writeEntry(fileA, new Entry(timestamp, 1, tags)); fileA.write(new Entry(timestamp, 1, tags));
final PdbFile fileB = tagsToFile.getFile(timestamp, tags); final PdbWriter fileB = tagsToFile.getWriter(timestamp, tags);
PdbWriter.writeEntry(fileA, new Entry(timestamp, 2, tags)); fileA.write(new Entry(timestamp, 2, tags));
Assert.assertEquals(fileA, fileB); Assert.assertEquals(fileA, fileB);
} }