andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

finalize refactoring

Browse Source
This commit is contained in:
Andreas Huber
2016-12-29 18:27:15 +01:00
parent 68ac1dd631
commit de241ceb6d
20 changed files with 287 additions and 571 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java
View File

@@ -1,27 +0,0 @@
package org.lucares.performance.db;
class BitFiddling {
static byte long3(final long x) {
return (byte) (x >> 24);
}
static byte long2(final long x) {
return (byte) (x >> 16);
}
static byte long1(final long x) {
return (byte) (x >> 8);
}
static byte long0(final long x) {
return (byte) (x);
}
static long makeLong(final byte b3, final byte b2, final byte b1, final byte b0) {
return ((((long) b3 & 0xff) << 24) | //
(((long) b2 & 0xff) << 16) | //
(((long) b1 & 0xff) << 8) | //
(((long) b0 & 0xff)));//
}
}

18
performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java
View File

@@ -1,9 +1,11 @@
package org.lucares.performance.db;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -22,7 +24,19 @@ public interface CollectionUtils {
return result;
}
public default <T> List<T> filter(final Collection<T> list, final Predicate<T> predicate) {
return list.stream().filter(predicate).collect(Collectors.toList());
public default <T> List<T> filter(final Collection<T> collection, final Predicate<T> predicate) {
return collection.stream().filter(predicate).collect(Collectors.toList());
}
public default <T, R> List<R> map(final Collection<T> collection, final Function<T, R> mapper) {
return collection.stream().map(mapper).collect(Collectors.toList());
}
public default <T> List<T> sorted(final Collection<T> collection, final Comparator<T> comparator) {
return collection.stream().sorted(comparator).collect(Collectors.toList());
}
public default <T> Optional<T> findFirst(final Collection<T> collection) {
return collection.stream().findFirst();
}
}

40
performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java
View File

@@ -1,34 +1,11 @@
package org.lucares.performance.db;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQuery;
import java.util.Calendar;
import java.util.TimeZone;
public class DateUtils {
private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC");
public static long getDateOffset(final OffsetDateTime date) {
return date.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli();
}
public static Calendar getCalendar() {
return Calendar.getInstance(TIME_ZONE_UTC);
}
public static OffsetDateTime getLastInstantOfDay(final OffsetDateTime date) {
return date.truncatedTo(ChronoUnit.DAYS).plusDays(1).minusNanos(1);
}
public static OffsetDateTime getDate(final int year, final int month, final int day, final int hour,
final int minute, final int second) {
@@ -40,18 +17,7 @@ public class DateUtils {
return OffsetDateTime.now(ZoneOffset.UTC);
}
public static OffsetDateTime parseAtZoneOffset(final String text, final DateTimeFormatter formatter,
final ZoneOffset zoneOffset) {
final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
@Override
public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
final LocalDate localDate = LocalDate.from(temporal);
final LocalTime localTime = LocalTime.from(temporal);
return OffsetDateTime.of(localDate, localTime, zoneOffset);
}
};
final OffsetDateTime date = formatter.parse(text, query);
return date;
public static OffsetDateTime epochMilliInUTC(final long lastEpochMilli) {
return OffsetDateTime.ofInstant(Instant.ofEpochMilli(lastEpochMilli), ZoneOffset.UTC);
}
}

113
performanceDb/src/main/java/org/lucares/performance/db/Day.java
View File

@@ -1,113 +0,0 @@
package org.lucares.performance.db;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQuery;
class Day {
private final OffsetDateTime date;
public Day(final int year, final int month, final int day) {
date = OffsetDateTime.of(year, month, day, 0, 0, 0, 0, ZoneOffset.UTC);
}
public Day(final OffsetDateTime date) {
this.date = date.truncatedTo(ChronoUnit.DAYS);
}
public Day() {
date = OffsetDateTime.now(ZoneOffset.UTC);
}
public int getYear() {
return date.getYear();
}
public int getMonth() {
return date.getMonthValue();
}
public int getDay() {
return date.getDayOfMonth();
}
public long getOffsetInEpochMilli() {
return date.toInstant().toEpochMilli();
}
@Override
public String toString() {
return format("-");
}
public String format(final String separator) {
final String pattern = createPattern(separator);
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
return date.format(formatter);
}
private static String createPattern(final String separator) {
return String.format("yyyy%1$sMM%1$sdd", separator);
}
public String serialize() {
return format("-");
}
public static Day fromString(final String dateOffset) {
final String pattern = createPattern("-");
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern);
final TemporalQuery<OffsetDateTime> query = new TemporalQuery<OffsetDateTime>() {
@Override
public OffsetDateTime queryFrom(final TemporalAccessor temporal) {
final LocalDate localDate = LocalDate.from(temporal);
final LocalTime localTime = LocalTime.MIDNIGHT;
return OffsetDateTime.of(localDate, localTime, ZoneOffset.UTC);
}
};
final OffsetDateTime date = formatter.parse(dateOffset, query);
return new Day(date);
}
public TimeRange toTimeRange() {
final OffsetDateTime from = date;
final OffsetDateTime to = DateUtils.getLastInstantOfDay(from);
return new TimeRange(from, to);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((date == null) ? 0 : date.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Day other = (Day) obj;
if (date == null) {
if (other.date != null)
return false;
} else if (!date.equals(other.date))
return false;
return true;
}
}

27
performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java
View File

@@ -1,27 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.OffsetDateTime;
import org.lucares.pdb.api.Tags;
import org.lucares.performance.db.PdbWriterManager.PdbFileSupplier;
class FileSupplier implements PdbFileSupplier {
private final TagsToFile tagsToFile;
public FileSupplier(final TagsToFile tagsToFile) {
super();
this.tagsToFile = tagsToFile;
}
@Override
public PdbFile supply(final Tags tags, final OffsetDateTime date) {
try {
final PdbFile pdbFile = tagsToFile.getFile(date, tags);
return pdbFile;
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}

14
performanceDb/src/main/java/org/lucares/performance/db/Group.java
View File

@@ -7,19 +7,23 @@ import org.lucares.pdb.api.Tags;
class Group {
private final Tags tags;
private final List<PdbFile> files;
private final List<PdbReader> readers;
public Group(final Tags tags, final List<PdbFile> files) {
public Group(final Tags tags, final List<PdbReader> files) {
super();
this.tags = tags;
this.files = files;
this.readers = files;
}
public Tags getTags() {
return tags;
}
public List<PdbFile> getFiles() {
return files;
public List<PdbReader> getReaders() {
return readers;
}
public void addReader(final PdbReader pdbReader) {
readers.add(pdbReader);
}
}

14
performanceDb/src/main/java/org/lucares/performance/db/Grouping.java
View File

@@ -21,23 +21,23 @@ public class Grouping {
this.groups.addAll(groups);
}
public static Grouping groupBy(final List<PdbFile> pdbFiles, final String groupByField) {
public static Grouping groupBy(final List<PdbReader> pdbReaders, final String groupByField) {
final Grouping result;
if (groupByField == NO_GROUPING) {
final Group group = new Group(null, pdbFiles);
final Group group = new Group(null, pdbReaders);
result = new Grouping(group);
} else {
final Map<String, Group> grouping = new HashMap<>();
for (final PdbFile pdbFile : pdbFiles) {
final Tags tags = pdbFile.getTags();
for (final PdbReader pdbReader : pdbReaders) {
final Tags tags = pdbReader.getPdbFile().getTags();
final String value = tags.getValue(groupByField);
if (value != null) {
addIfNotExists(grouping, groupByField, value);
grouping.get(value).getFiles().add(pdbFile);
grouping.get(value).addReader(pdbReader);
}
}
result = new Grouping(grouping.values());
@@ -49,9 +49,9 @@ public class Grouping {
final String value) {
if (!grouping.containsKey(value)) {
final Tags tags = Tags.create(groupByField, value);
final List<PdbFile> files = new ArrayList<>();
final List<PdbReader> readers = new ArrayList<>();
grouping.put(value, new Group(tags, files));
grouping.put(value, new Group(tags, readers));
}
}

10
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -1,6 +1,8 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
@@ -69,4 +71,12 @@ class PdbFile {
return false;
return true;
}
public boolean exists() throws ReadException {
try {
return Files.isRegularFile(path) && Files.size(path) >= ByteType.VersionByte.MIN_LENGTH;
} catch (final IOException e) {
throw new ReadException(e);
}
}
}

18
performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java
View File

@@ -1,18 +0,0 @@
package org.lucares.performance.db;
import java.time.OffsetDateTime;
import java.util.Comparator;
public class PdbFileByTimeAsc implements Comparator<PdbFileOffsetTime> {
public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc();
@Override
public int compare(final PdbFileOffsetTime o1, final PdbFileOffsetTime o2) {
final OffsetDateTime o1From = o1.getOffsetTime();
final OffsetDateTime o2From = o2.getOffsetTime();
return o1From.compareTo(o2From);
}
}

45
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -1,31 +1,23 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
import java.util.Queue;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbFileIterator.class.getCanonicalName());
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles;
private final Queue<PdbReader> pdbFiles;
private PdbReader reader;
private PdbFile currentPdbFile;
public EntrySupplier(final Collection<PdbFile> pdbFiles) {
public EntrySupplier(final Collection<PdbReader> pdbFiles) {
super();
this.pdbFiles = new ArrayDeque<>(pdbFiles);
}
@@ -39,14 +31,14 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader == null) {
return null;
}
final Entry entry = reader.readNullableEntry(currentPdbFile.getTags());
final Entry entry = reader.readNullableEntry(reader.getPdbFile().getTags());
if (entry == null) {
nextFile();
if (reader == null) {
return null;
} else {
final Tags tags = currentPdbFile.getTags();
final Tags tags = reader.getPdbFile().getTags();
return reader.readEntry(tags).orElse(null);
}
}
@@ -62,22 +54,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
reader = null;
}
while (!pdbFiles.isEmpty()) {
currentPdbFile = pdbFiles.poll();
try {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.log(Level.WARNING, "the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
}
reader = pdbFiles.poll();
}
@Override
@@ -85,6 +62,14 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
if (reader != null) {
reader.close();
}
while (!pdbFiles.isEmpty()) {
try {
pdbFiles.poll().close();
} catch (final Exception e) {
e.printStackTrace();
}
}
}
}
@@ -93,7 +78,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) {
public PdbFileIterator(final Collection<PdbReader> pdbFiles) {
supplier = new EntrySupplier(pdbFiles);
}
@@ -117,7 +102,7 @@ public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
}
@Override
public void close() throws Exception {
public void close() {
supplier.close();
}

15
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -25,13 +25,21 @@ class PdbReader implements AutoCloseable {
private long index = 0;
private int peekedByte = PEEK_NOT_SET;
public PdbReader(final PdbFile pdbFile) throws FileNotFoundException {
private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws ReadException {
super();
try {
this.pdbFile = pdbFile;
final File storageFile = pdbFile.getPath().toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile));
init();
} catch (final FileNotFoundException e) {
throw new ReadException(e);
}
}
private void init() {
@@ -49,6 +57,10 @@ class PdbReader implements AutoCloseable {
}
}
public PdbFile getPdbFile() {
return pdbFile;
}
/**
* Seek to the end of the file.
*
@@ -161,5 +173,4 @@ class PdbReader implements AutoCloseable {
final long bytePrefix = ByteType.CONTINUATION.getBytePrefix();
return bytePrefix == (nextByte & bytePrefix);
}
}

14
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -3,10 +3,10 @@ package org.lucares.performance.db;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.time.OffsetDateTime;
import java.util.logging.Logger;
import org.lucares.pdb.api.Entry;
@@ -68,9 +68,7 @@ import org.lucares.pdb.api.Entry;
* 1 and has room for 7 bits. The result looks like this: <b>01</b><i>000001<i>
* <b>1</b><i>1001010</i>
*/
class PdbWriter implements AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbWriter.class.getCanonicalName());
class PdbWriter implements AutoCloseable, Flushable {
private static final boolean APPEND = true;
@@ -100,10 +98,14 @@ class PdbWriter implements AutoCloseable {
}
}
public PdbFile getFile() {
public PdbFile getPdbFile() {
return pdbFile;
}
public OffsetDateTime getDateOffset() {
return DateUtils.epochMilliInUTC(lastEpochMilli);
}
public void write(final Entry entry) throws WriteException {
System.out.println(entry);
final long epochMilli = entry.getEpochMilli();
@@ -137,6 +139,7 @@ class PdbWriter implements AutoCloseable {
outputStream.close();
}
@Override
public void flush() throws IOException {
outputStream.flush();
}
@@ -172,4 +175,5 @@ class PdbWriter implements AutoCloseable {
public static void init(final PdbFile result) throws IOException {
writeEntry(result);
}
}

18
performanceDb/src/main/java/org/lucares/performance/db/PdbWriterByTimeAsc.java Normal file
View File

@@ -0,0 +1,18 @@
package org.lucares.performance.db;
import java.time.OffsetDateTime;
import java.util.Comparator;
public class PdbWriterByTimeAsc implements Comparator<PdbWriter> {
public static final PdbWriterByTimeAsc INSTANCE = new PdbWriterByTimeAsc();
@Override
public int compare(final PdbWriter o1, final PdbWriter o2) {
final OffsetDateTime o1From = o1.getDateOffset();
final OffsetDateTime o2From = o2.getDateOffset();
return o1From.compareTo(o2From);
}
}

87
performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java
View File

@@ -1,87 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.lucares.pdb.api.Tags;
public class PdbWriterManager implements AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName());
public interface PdbFileSupplier {
public PdbFile supply(Tags tags, OffsetDateTime date);
}
final Map<PdbFile, PdbWriter> map = new HashMap<>();
private final PdbFileSupplier supplier;
private Day lastDay = new Day(OffsetDateTime.MIN);
public PdbWriterManager(final PdbFileSupplier supplier) {
this.supplier = supplier;
}
public PdbWriter getWriter(final Tags tags, final OffsetDateTime date) throws IOException {
handleDateChange(date);
final PdbFile pdbFile = supplier.supply(tags, date);
if (!map.containsKey(pdbFile)) {
final PdbWriter writer = new PdbWriter(pdbFile);
map.put(pdbFile, writer);
}
return map.get(pdbFile);
}
private void handleDateChange(final OffsetDateTime date) {
final Day day = new Day(date);
if (!day.equals(lastDay)) {
LOGGER.info("finished with " + day);
closeFiles();
lastDay = day;
}
}
public void flush() {
LOGGER.info("flushing all files");
for (final PdbWriter writer : map.values()) {
try {
writer.flush();
} catch (final IOException e) {
LOGGER.log(Level.WARNING, e.getMessage(), e);
}
}
}
@Override
public void close() {
closeFiles();
}
private void closeFiles() {
final Iterator<Entry<PdbFile, PdbWriter>> it = map.entrySet().iterator();
while (it.hasNext()) {
final Entry<PdbFile, PdbWriter> entry = it.next();
final PdbWriter writer = entry.getValue();
try {
writer.close();
} catch (final IOException e) {
LOGGER.log(Level.WARNING, e.getMessage(), e);
}
it.remove();
}
}
}

31
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -1,7 +1,6 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
@@ -61,7 +60,7 @@ public class PerformanceDb implements AutoCloseable {
long count = 0;
double durationInManager = 0;
try (final PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
try {
long start = System.nanoTime();
while (true) {
@@ -75,7 +74,7 @@ public class PerformanceDb implements AutoCloseable {
final Tags tags = entry.getTags();
final OffsetDateTime date = entry.getDate();
final PdbWriter writer = manager.getWriter(tags, date);
final PdbWriter writer = tagsToFile.getWriter(date, tags);
writer.write(entry);
count++;
@@ -94,13 +93,13 @@ public class PerformanceDb implements AutoCloseable {
}
}
} catch (final IOException e) {
} catch (final RuntimeException e) {
throw new WriteException(e);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.info("Thread was interrupted. Aborting exectution.");
} finally {
//
tagsToFile.flush();
}
}
@@ -123,9 +122,9 @@ public class PerformanceDb implements AutoCloseable {
*/
public Result get(final String query, final String groupBy) {
final List<PdbFile> pdbFiles = tagsToFile.getFilesForQuery(query);
final List<PdbReader> pdbReaders = tagsToFile.getReaders(query);
final Grouping grouping = Grouping.groupBy(pdbFiles, groupBy);
final Grouping grouping = Grouping.groupBy(pdbReaders, groupBy);
final Result result = toResult(grouping);
@@ -135,7 +134,7 @@ public class PerformanceDb implements AutoCloseable {
private Result toResult(final Grouping grouping) {
final List<GroupResult> groupResults = new ArrayList<>();
for (final Group group : grouping.getGroups()) {
final Stream<Entry> stream = toStream(group.getFiles());
final Stream<Entry> stream = toStream(group.getReaders());
final GroupResult groupResult = new GroupResult(stream, group.getTags());
groupResults.add(groupResult);
}
@@ -143,11 +142,19 @@ public class PerformanceDb implements AutoCloseable {
return result;
}
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final Iterator<Entry> iterator = new PdbFileIterator(pdbFiles);
private Stream<Entry> toStream(final List<PdbReader> pdbFiles) {
final PdbFileIterator iterator = new PdbFileIterator(pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
return StreamSupport.stream(spliterator, false);
final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
final Stream<Entry> result = stream.onClose(() -> {
try {
iterator.close();
} catch (final RuntimeException e) {
e.printStackTrace();
}
});
return result;
}
@Override
@@ -158,6 +165,8 @@ public class PerformanceDb implements AutoCloseable {
// H2 doesn't actually do anything in close
throw new IllegalStateException(e);
}
tagsToFile.close();
}
public List<Proposal> autocomplete(final String query, final int caretIndex) {

160
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,21 +1,30 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.Flushable;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import org.lucares.ludb.Document;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Tags;
public class TagsToFile implements CollectionUtils {
public class TagsToFile implements CollectionUtils, AutoCloseable {
private static final Logger LOGGER = Logger.getLogger(TagsToFile.class.getCanonicalName());
private static class TagSpecificBaseDir {
private final Path path;
@@ -37,9 +46,36 @@ public class TagsToFile implements CollectionUtils {
}
}
private static class WriterCache {
final List<PdbWriter> writers = new ArrayList<>();
public List<PdbWriter> getWriters() {
return writers;
}
public void addWriter(final PdbWriter writer) {
writers.add(writer);
}
public Optional<PdbWriter> writer(final PdbFile pdbFile) {
return writers.stream().filter(w -> Objects.equals(w.getPdbFile(), pdbFile)).findAny();
}
public Optional<Path> tagSpecificBaseDir() {
if (writers.size() > 0) {
return Optional.of(writers.get(0).getPdbFile().getPath().getParent());
}
return Optional.empty();
}
}
private final H2DB db;
private final Path dataDirectory;
private final Map<Tags, WriterCache> cachedWriters = new HashMap<>();
public TagsToFile(final Path dataDirectory, final H2DB db) {
this.dataDirectory = dataDirectory;
this.db = db;
@@ -57,7 +93,20 @@ public class TagsToFile implements CollectionUtils {
return getFilesForQuery(query);
}
List<PdbFile> getFilesForQuery(final String query) {
public List<PdbReader> getReaders(final String query) {
final List<PdbReader> result = new ArrayList<>();
final List<PdbFile> filesForQuery = getFilesForQuery(query);
for (final PdbFile pdbFile : filesForQuery) {
final PdbReader reader = new PdbReader(pdbFile);
result.add(reader);
}
return result;
}
public List<PdbFile> getFilesForQuery(final String query) {
final List<PdbFile> result = new ArrayList<>();
final List<TagSpecificBaseDir> tagSpecificFolders = getTagSpecificFolders(query);
@@ -107,46 +156,55 @@ public class TagsToFile implements CollectionUtils {
return tagsOfFile;
}
public PdbFile getFile(final OffsetDateTime date, final Tags tags) throws FileNotFoundException, IOException {
public PdbWriter getWriter(final OffsetDateTime date, final Tags tags) throws ReadException, WriteException {
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
final List<PdbFileOffsetTime> preResult = new ArrayList<>();
assertAllFilesHaveSameFolder(pdbFiles);
PdbFile result;
for (final PdbFile pdbFile : pdbFiles) {
final WriterCache writersForTags = getOrInit(tags);
if (Files.isRegularFile(pdbFile.getPath())
&& Files.size(pdbFile.getPath()) >= ByteType.VersionByte.MIN_LENGTH) {
pdbFiles.removeIf(f -> !f.exists());
final List<Optional<PdbWriter>> optionalWriters = map(pdbFiles, writersForTags::writer);
final List<Optional<PdbWriter>> existingWriters = filter(optionalWriters, Optional::isPresent);
final List<PdbWriter> writers = map(existingWriters, Optional::get);
final List<PdbWriter> candidateWriters = filter(writers, writer -> {
final OffsetDateTime offsetTime = writer.getDateOffset();
return !date.isBefore(offsetTime);
});
final List<PdbWriter> sortedCanditateWriters = sorted(candidateWriters, PdbWriterByTimeAsc.INSTANCE.reversed());
final Optional<PdbWriter> optionalFirst = findFirst(sortedCanditateWriters);
final OffsetDateTime offsetTime = PdbFileUtils.dateOffset(pdbFile);
if (!offsetTime.isAfter(date)) {
preResult.add(new PdbFileOffsetTime(pdbFile, offsetTime));
}
}
}
if (preResult.isEmpty()) {
Path tagSpecificStorageFolder;
if (pdbFiles.isEmpty()) {
tagSpecificStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
} else {
final Path storageFilePath = pdbFiles.get(0).getPath();
tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
}
result = createNewPdbFile(tags, tagSpecificStorageFolder);
} else {
Collections.sort(preResult, PdbFileByTimeAsc.INSTANCE.reversed());
result = preResult.get(0).getPdbFile();
}
final PdbWriter result = optionalFirst.orElseGet(() -> newPdbWriter(tags));
return result;
}
private WriterCache getOrInit(final Tags tags) {
if (!cachedWriters.containsKey(tags)) {
cachedWriters.put(tags, new WriterCache());
}
return cachedWriters.get(tags);
}
private PdbWriter newPdbWriter(final Tags tags) {
try {
PdbWriter result;
final Path tagSpecificStorageFolder = getOrInit(tags).tagSpecificBaseDir()
.orElse(StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags));
final PdbFile pdbFile = createNewPdbFile(tags, tagSpecificStorageFolder);
result = new PdbWriter(pdbFile);
getOrInit(tags).addWriter(result);
return result;
} catch (final IOException e) {
throw new WriteException(e);
}
}
private void assertAllFilesHaveSameFolder(final List<PdbFile> pdbFiles) {
final Set<Path> reducedFolder = pdbFiles.stream()//
.map(PdbFile::getPath)//
@@ -190,4 +248,40 @@ public class TagsToFile implements CollectionUtils {
return result;
}
@SuppressWarnings("unchecked")
private <T extends AutoCloseable & Flushable> void forEachWriter(final Consumer<T> consumer) {
for (final Entry<Tags, WriterCache> readersWriters : cachedWriters.entrySet()) {
for (final PdbWriter writer : readersWriters.getValue().getWriters()) {
try {
consumer.accept((T) writer);
} catch (final RuntimeException e) {
LOGGER.log(Level.WARNING, "failed to close writer for file " + writer.getPdbFile().getPath(), e);
}
}
}
}
@Override
public void close() {
forEachWriter(t -> {
try {
t.close();
} catch (final Exception e) {
throw new WriteException(e);
}
});
}
public void flush() {
forEachWriter(t -> {
try {
t.flush();
} catch (final IOException e) {
throw new WriteException(e);
}
});
}
}

42
performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java
View File

@@ -1,42 +0,0 @@
package org.lucares.performance.db;
import org.testng.annotations.Test;
@Test
public class BitFiddlingTest {
// TODO @ahr remove or move
// public void testEncodingMeasurement() throws Exception {
//
// final List<ByteType> types = Arrays.asList(ByteType.DATE_INCREMENT,
// ByteType.DATE_OFFSET, ByteType.MEASUREMENT,
// ByteType.VERSION);
//
// final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L,
// 202L, 255L, 256L, 8191L, 8192L, 1048575L,
// 1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L,
// 2199023255552L, 281474976710655L,
// 281474976710656L, 36028797018963967L, 36028797018963968L,
// 4611686018427387901L, 4611686018427387904L);
//
// for (final Long value : values) {
// for (final ByteType type : types) {
// encodeDecode(value, type);
// }
// }
// }
//
// private void encodeDecode(final Long value, final ByteType byteType)
// throws Exception {
//
// final ByteArrayOutputStream output = new ByteArrayOutputStream();
// BitFiddling.writeValue(value, byteType, output);
//
// final byte[] byteArray = output.toByteArray();
// final ByteArrayInputStream input = new ByteArrayInputStream(byteArray);
// final Long readValue = BitFiddling.readValue(byteType, input);
//
// Assert.assertEquals(readValue, value);
// }
}

82
performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java
View File

@@ -1,82 +0,0 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class PdbWriterManagerTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@Test
public void testManager() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final Tags tagsA = Tags.create("key", "A");
final Tags tagsB = Tags.create("key", "B");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime date = OffsetDateTime.now();
final PdbWriter firstWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter secondWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter firstWriterForTagsB = manager.getWriter(tagsB, date);
Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
}
}
}
@Test
public void testManager2() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final Tags tags = Tags.create("key", "A");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime morning = OffsetDateTime.of(2016, 1, 1, 8, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime noon = OffsetDateTime.of(2016, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime afternoon = OffsetDateTime.of(2016, 1, 1, 17, 0, 0, 0, ZoneOffset.UTC);
final PdbWriter writerNoon = manager.getWriter(tags, noon);
writerNoon.write(new Entry(noon, 1, tags));
final PdbWriter writerMorning = manager.getWriter(tags, morning);
writerMorning.write(new Entry(morning, 2, tags));
final PdbWriter writerAfternoon = manager.getWriter(tags, afternoon);
writerAfternoon.write(new Entry(afternoon, 3, tags));
Assert.assertSame(writerNoon, writerAfternoon);
Assert.assertNotSame(writerNoon, writerMorning);
}
}
}
}

1
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -163,7 +163,6 @@ public class PerformanceDbTest {
actualEntriesAll.sort(EntryByDateComparator.INSTANCE);
Assert.assertEquals(actualEntriesAll, expectedAll);
}
}

76
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -32,49 +32,44 @@ public class TagsToFilesTest {
public void test() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final OffsetDateTime date = OffsetDateTime.now(ZoneOffset.UTC);
final Tags tags = Tags.create("myKey", "myValue");
final PdbFile newFileForTags = tagsToFile.getFile(date, tags);
PdbWriter.writeEntry(newFileForTags);
final PdbWriter newFileForTags = tagsToFile.getWriter(date, tags);
final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
final PdbWriter existingFileForTags = tagsToFile.getWriter(date, tags);
Assert.assertEquals(newFileForTags, existingFileForTags);
Assert.assertSame(newFileForTags, existingFileForTags);
}
}
public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final OffsetDateTime day1 = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final OffsetDateTime day2 = DateUtils.getDate(2016, 1, 2, 1, 1, 1);
final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileForDay1 = tagsToFile.getFile(day1, tags);
final PdbFile fileForDay2 = tagsToFile.getFile(day2, tags);
final PdbWriter writerForDay1 = tagsToFile.getWriter(day1, tags);
writerForDay1.write(new Entry(day1, 1, tags));
final PdbWriter writerForDay2 = tagsToFile.getWriter(day2, tags);
writerForDay2.write(new Entry(day2, 2, tags));
Assert.assertEquals(fileForDay1, fileForDay2);
final PdbFile existingFileForDay1 = tagsToFile.getFile(day1, tags);
Assert.assertEquals(fileForDay1, existingFileForDay1);
Assert.assertSame(writerForDay1, writerForDay2);
}
}
@Test(invocationCount = 1)
public void testNewFileIfDateIsTooOld() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final OffsetDateTime afternoon = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final OffsetDateTime morning = DateUtils.getDate(2016, 1, 1, 12, 1, 1);
@@ -83,43 +78,46 @@ public class TagsToFilesTest {
final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileAfternoon = tagsToFile.getFile(afternoon, tags);
PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags));
final PdbWriter writerAfternoon = tagsToFile.getWriter(afternoon, tags);
writerAfternoon.write(new Entry(afternoon, 1, tags));
final PdbFile fileMorning = tagsToFile.getFile(morning, tags);
PdbWriter.writeEntry(fileMorning, new Entry(morning, 2, tags));
final PdbWriter writerMorning = tagsToFile.getWriter(morning, tags);
writerMorning.write(new Entry(morning, 2, tags));
Assert.assertNotEquals(fileAfternoon, fileMorning);
Assert.assertNotSame(writerAfternoon, writerMorning);
Assert.assertNotEquals(writerAfternoon.getPdbFile(), writerMorning.getPdbFile());
final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags);
PdbWriter.writeEntry(fileEarlyMorning, new Entry(earlyMorning, 3, tags));
final PdbWriter writerEarlyMorning = tagsToFile.getWriter(earlyMorning, tags);
writerEarlyMorning.write(new Entry(earlyMorning, 3, tags));
Assert.assertNotEquals(fileEarlyMorning, fileAfternoon);
Assert.assertNotEquals(fileEarlyMorning, fileMorning);
Assert.assertNotSame(writerEarlyMorning, writerAfternoon);
Assert.assertNotSame(writerEarlyMorning, writerMorning);
final PdbFile fileEvening = tagsToFile.getFile(evening, tags);
final PdbWriter writerEvening = tagsToFile.getWriter(evening, tags);
Assert.assertEquals(fileEvening, fileAfternoon, "the evening event can be appended to the afternoon file");
Assert.assertNotEquals(fileEvening, fileMorning);
Assert.assertNotEquals(fileEvening, fileEarlyMorning);
Assert.assertSame(writerEvening, writerAfternoon,
"the evening event can be appended to the afternoon file");
Assert.assertNotSame(writerEvening, writerMorning);
Assert.assertNotSame(writerEvening, writerEarlyMorning);
Assert.assertNotEquals(writerEvening.getPdbFile(), writerMorning.getPdbFile());
Assert.assertNotEquals(writerEvening.getPdbFile(), writerEarlyMorning.getPdbFile());
}
}
public void testIdenticalDatesGoIntoSameFile() throws Exception {
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);) {
final OffsetDateTime timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1);
final Tags tags = Tags.create("myKey", "myValue");
final PdbFile fileA = tagsToFile.getFile(timestamp, tags);
PdbWriter.writeEntry(fileA, new Entry(timestamp, 1, tags));
final PdbWriter fileA = tagsToFile.getWriter(timestamp, tags);
fileA.write(new Entry(timestamp, 1, tags));
final PdbFile fileB = tagsToFile.getFile(timestamp, tags);
PdbWriter.writeEntry(fileA, new Entry(timestamp, 2, tags));
final PdbWriter fileB = tagsToFile.getWriter(timestamp, tags);
fileA.write(new Entry(timestamp, 2, tags));
Assert.assertEquals(fileA, fileB);
}