inital commit

This commit is contained in:
2016-12-04 10:41:37 +01:00
commit 256b278428
34 changed files with 2011 additions and 0 deletions

6
performanceDb/.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
/bin/
/build/
/.settings/
/.classpath
/.project
/test-output

View File

@@ -0,0 +1,7 @@
dependencies {
//compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5'
compile 'org.lucares:ludb:1.0.20161002111352'
//compile 'commons-io:commons-io:2.5'
}

View File

@@ -0,0 +1,27 @@
package org.lucares.performance.db;
class BitFiddling {
static byte long3(final long x) {
return (byte) (x >> 24);
}
static byte long2(final long x) {
return (byte) (x >> 16);
}
static byte long1(final long x) {
return (byte) (x >> 8);
}
static byte long0(final long x) {
return (byte) (x);
}
static long makeLong(final byte b3, final byte b2, final byte b1, final byte b0) {
return ((((long) b3 & 0xff) << 24) | //
(((long) b2 & 0xff) << 16) | //
(((long) b1 & 0xff) << 8) | //
(((long) b0 & 0xff)));//
}
}

View File

@@ -0,0 +1,27 @@
package org.lucares.performance.db;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public interface CollectionUtils {
public default <T, V> Map<T, V> toMap(final Iterable<V> iterable, final Function<V, T> keyMapper) {
final Map<T, V> result = new HashMap<>();
for (final V value : iterable) {
final T key = keyMapper.apply(value);
result.put(key, value);
}
return result;
}
public default <T> List<T> filter(final List<T> list, final Predicate<T> predicate) {
return list.stream().filter(predicate).collect(Collectors.toList());
}
}

View File

@@ -0,0 +1,83 @@
package org.lucares.performance.db;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
public class DateUtils {
private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC");
private static final SimpleDateFormat YEAR = new SimpleDateFormat("yyyy");
private static final SimpleDateFormat MONTH = new SimpleDateFormat("MM");
private static final SimpleDateFormat DAY = new SimpleDateFormat("dd");
public static final synchronized Day getDay(final Date date) {
final String yearString = YEAR.format(date);
final String monthString = MONTH.format(date);
final String dayString = DAY.format(date);
final int year = Integer.parseInt(yearString, 10);
final int month = Integer.parseInt(monthString, 10);
final int day = Integer.parseInt(dayString, 10);
return new Day(year, month, day);
}
public static long getDateOffset(final Date date) {
return getMidnightSameDay(date).getTime();
}
public static Calendar getCalendar() {
return Calendar.getInstance(TIME_ZONE_UTC);
}
public static Date getMidnightSameDay(final Date date) {
final Calendar exactTime = getCalendar();
exactTime.setTime(date);
final Calendar midnight = getCalendar();
final int year = exactTime.get(Calendar.YEAR);
final int month = exactTime.get(Calendar.MONTH);
final int day = exactTime.get(Calendar.DATE);
midnight.set(year, month, day);
midnight.set(Calendar.HOUR, 0);
midnight.set(Calendar.MINUTE, 0);
midnight.set(Calendar.SECOND, 0);
midnight.set(Calendar.MILLISECOND, 0);
return midnight.getTime();
}
public static Date getMidnightNextDay(final Date date) {
final Calendar exactTime = Calendar.getInstance(TIME_ZONE_UTC);
exactTime.setTime(date);
exactTime.add(Calendar.DATE, 1);
exactTime.add(Calendar.MILLISECOND, -1);
return exactTime.getTime();
}
public static Date getDate(final int year, final int month, final int day, final int hour, final int minute,
final int second) {
final Calendar calendar = getCalendar();
calendar.set(year, month - 1, day);
calendar.set(Calendar.HOUR, hour);
calendar.set(Calendar.MINUTE, minute);
calendar.set(Calendar.SECOND, second);
calendar.set(Calendar.MILLISECOND, 0);
return calendar.getTime();
}
public static String format(final Date date) {
final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH-mm:ss,SSS Z");
dateFormat.setTimeZone(TIME_ZONE_UTC);
return dateFormat.format(date);
}
}

View File

@@ -0,0 +1,36 @@
package org.lucares.performance.db;
class Day {
private final int year;
private final int month;
private final int day;
public Day(final int year, final int month, final int day) {
super();
this.year = year;
this.month = month;
this.day = day;
}
public int getYear() {
return year;
}
public int getMonth() {
return month;
}
public int getDay() {
return day;
}
@Override
public String toString() {
return format("-");
}
public String format(final String separator) {
return String.format("%04d%s%02d%s%02d", year, separator, month, separator, day);
}
}

View File

@@ -0,0 +1,56 @@
package org.lucares.performance.db;
import java.util.Date;
public class Entry {
private final Date date;
private final long value;
public Entry(final Date date, final long value) {
super();
this.date = date;
this.value = value;
}
public Date getDate() {
return date;
}
public long getValue() {
return value;
}
@Override
public String toString() {
return DateUtils.format(date) + " = " + value;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((date == null) ? 0 : date.hashCode());
result = prime * result + (int) (value ^ (value >>> 32));
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Entry other = (Entry) obj;
if (date == null) {
if (other.date != null)
return false;
} else if (!date.equals(other.date))
return false;
if (value != other.value)
return false;
return true;
}
}

View File

@@ -0,0 +1,23 @@
package org.lucares.performance.db;
class Fields {
static final String TAG_PREFIX = "__tag_";
static final String DATE_OFFSET = "__date_offset__";
static String prefixedKey(final String key) {
return TAG_PREFIX + key;
}
static boolean isPrefixedKey(final String key) {
return key.startsWith(TAG_PREFIX);
}
static String stripPrefix(final String key) {
if (!isPrefixedKey(key)) {
throw new IllegalArgumentException(key + " is not prefixed by " + TAG_PREFIX);
}
return key.substring(TAG_PREFIX.length());
}
}

View File

@@ -0,0 +1,55 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.logging.Level;
import java.util.logging.Logger;
public class FileUtils {
private static final Logger LOGGER = Logger.getLogger(FileUtils.class.getCanonicalName());
private static final class RecursiveDeleter extends SimpleFileVisitor<Path> {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
Files.delete(file);
LOGGER.info("deleted: " + file.toFile().getAbsolutePath());
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
Files.delete(dir);
LOGGER.info("deleted: " + dir.toFile().getAbsolutePath());
return FileVisitResult.CONTINUE;
}
}
public static void delete(final Path path) {
final int maxAttempts = 10;
int attempt = 1;
while (attempt <= maxAttempts) {
try {
LOGGER.info(
"deleting '" + path.toFile().getAbsolutePath() + "' attempt " + attempt + " of " + maxAttempts);
Files.walkFileTree(path, new RecursiveDeleter());
break;
} catch (final IOException e) {
final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt
+ " of " + maxAttempts;
LOGGER.log(Level.WARNING, msg, e);
}
attempt++;
}
}
}

View File

@@ -0,0 +1,91 @@
package org.lucares.performance.db;
import java.io.File;
import java.util.Date;
class PdbFile {
private final Tags tags;
private final long dateOffset;
private final File file;
public PdbFile(final long dateOffset, final File file, final Tags tags) {
checkOffset(dateOffset);
this.dateOffset = dateOffset;
this.file = file;
this.tags = tags;
}
public static PdbFile today(final File file, final Tags tags) {
final long dateOffset = DateUtils.getDateOffset(new Date());
return new PdbFile(dateOffset, file, tags);
}
private void checkOffset(final long dateOffset) {
final Date date = new Date(dateOffset);
final long expectedDateOffset = DateUtils.getDateOffset(date);
if (dateOffset != expectedDateOffset) {
throw new IllegalArgumentException("dateOffset must be at exactly midnight UTC: " + dateOffset + " != "
+ expectedDateOffset + "(" + date + " != " + new Date(expectedDateOffset) + ")");
}
}
public Tags getTags() {
return tags;
}
public File getFile() {
return file;
}
public long getDateOffset() {
return dateOffset;
}
public TimeRange getTimeRange() {
final Date from = new Date(dateOffset);
final Date to = DateUtils.getMidnightNextDay(from);
return new TimeRange(from, to);
}
@Override
public String toString() {
return "PdbFile [" + file + " " + getTimeRange() + " " + tags + "]\n";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (dateOffset ^ (dateOffset >>> 32));
result = prime * result + ((file == null) ? 0 : file.hashCode());
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (dateOffset != other.dateOffset)
return false;
if (file == null) {
if (other.file != null)
return false;
} else if (!file.equals(other.file))
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
}

View File

@@ -0,0 +1,14 @@
package org.lucares.performance.db;
import java.util.Comparator;
public class PdbFileByTimeAsc implements Comparator<PdbFile> {
@Override
public int compare(final PdbFile o1, final PdbFile o2) {
final long difference = o1.getDateOffset() - o2.getDateOffset();
return difference < 0 ? -1 : (difference == 0 ? 0 : 1);
}
}

View File

@@ -0,0 +1,109 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
import java.util.Queue;
import java.util.function.Supplier;
import java.util.logging.Level;
import java.util.logging.Logger;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbFileIterator.class.getCanonicalName());
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles;
private PdbReader reader;
public EntrySupplier(final Collection<PdbFile> pdbFiles) {
super();
this.pdbFiles = new ArrayDeque<>(pdbFiles);
}
@Override
public Entry get() {
if (reader == null) {
nextFile();
}
final Optional<Entry> optionalEntry = reader.readEntry();
return optionalEntry.orElseGet(() -> {
nextFile();
if (reader == null) {
return null;
} else {
return reader.readEntry().orElse(null);
}
});
}
private void nextFile() {
if (reader != null) {
reader.close();
reader = null;
}
while (!pdbFiles.isEmpty()) {
final PdbFile pdbFile = pdbFiles.poll();
try {
if (pdbFile.getFile().length() > 0) {
reader = new PdbReader(pdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + pdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.log(Level.WARNING, "the pdbFile " + pdbFile.getFile() + " is missing", e);
}
}
}
@Override
public void close() {
if (reader != null) {
reader.close();
}
}
}
private final EntrySupplier supplier;
private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(pdbFiles);
}
@Override
public boolean hasNext() {
final boolean result;
if (next.isPresent()) {
result = true;
} else {
next = Optional.ofNullable(supplier.get());
result = next.isPresent();
}
return result;
}
@Override
public Entry next() {
final Entry result = next.orElseGet(supplier::get);
next = Optional.empty();
return result;
}
@Override
public void close() throws Exception {
supplier.close();
}
}

View File

@@ -0,0 +1,189 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.Date;
import java.util.Optional;
class PdbReader implements AutoCloseable {
private static final int BYTES_PER_VALUE = 4;
private final RandomAccessFile data;
private final byte[] buffer = new byte[BYTES_PER_VALUE];
private final PdbFile pdbFile;
public PdbReader(final PdbFile pdbFile) throws FileNotFoundException {
super();
this.pdbFile = pdbFile;
this.data = new RandomAccessFile(pdbFile.getFile(), "r");
}
/**
* Reads the next value.
* <p>
* All values are non-negative. A negative return value indicates that the
* end of the file has been reached
*
* @return the value or -1 if end of stream has been reached
*/
public long readValue() {
assertPositionIsAValuePosition();
return read();
}
/**
* Reads the next date value.
*
* @return the date, or {@code null} if end of stream has been reached
* @throws IOException
*/
public Date readDate() {
assertPositionIsADatePosition();
final long value = read();
if (value < 0) {
return null;
}
return new Date(pdbFile.getDateOffset() + value);
}
// visible for test
long read() {
try {
final int read = data.read(buffer);
if (read < 0) {
return -1;
}
if (read != BYTES_PER_VALUE) {
throw new IllegalStateException("invalid file");
}
return BitFiddling.makeLong(buffer[0], buffer[1], buffer[2], buffer[3]);
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private void assertPositionIsADatePosition() {
try {
assertPositionIsValid();
if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 0) {
throw new IllegalStateException("file pointer is not at a date position: " + data.getFilePointer());
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private void assertPositionIsAValuePosition() {
assertPositionIsValid();
try {
if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 1) {
throw new IllegalStateException("file pointer is not at a value position: " + data.getFilePointer());
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private void assertPositionIsValid() {
try {
if (data.getFilePointer() % BYTES_PER_VALUE != 0) {
throw new IllegalStateException("file pointer is at an illegal position. It is at "
+ data.getFilePointer() + " which is not divisible by " + BYTES_PER_VALUE);
}
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
/**
* Seek to the n-th value.
*
* @param n
*/
public void seek(final long n) {
try {
if (n < 0) {
throw new IllegalArgumentException("n must be non-negative, but was " + n);
}
final long pos = n * BYTES_PER_VALUE;
if (pos >= data.length()) {
throw new IllegalArgumentException("cannot seek to value " + n + ", because the file only has "
+ (data.length() / BYTES_PER_VALUE) + " values");
}
data.seek(pos);
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
/**
* Seek to the end of the file.
* <p>
* After this operation you can read {@code n} values.
*
* @param n
* seek to the n-th last value
* @throws ReadRuntimeException
* if the file does not have {@code n} entries
*/
public void seekTail(final long n) {
try {
if (n < 0) {
throw new IllegalArgumentException("n must be non-negative, but was " + n);
}
final long pos = computeSeekPosition(n);
data.seek(pos);
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
public boolean canSeekTail(final long n) {
try {
final long pos = computeSeekPosition(n);
return pos >= 0;
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
private long computeSeekPosition(final long n) throws IOException {
final long length = data.length();
final long pos = length - BYTES_PER_VALUE * n;
return pos;
}
@Override
public void close() {
try {
data.close();
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
public Optional<Entry> readEntry() throws ReadRuntimeException {
final Date date = readDate();
if (date == null) {
return Optional.empty();
}
final long value = readValue();
if (value < 0) {
return Optional.empty();
}
return Optional.of(new Entry(date, value));
}
}

View File

@@ -0,0 +1,67 @@
package org.lucares.performance.db;
import java.io.BufferedOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Date;
class PdbWriter implements AutoCloseable {
private static final long MAX_VALUE = 0xFF_FF_FF_FFL;
private static final boolean APPEND = true;
private final OutputStream outputStream;
private final PdbFile pdbFile;
PdbWriter(final PdbFile pdbFile) throws FileNotFoundException {
this.pdbFile = pdbFile;
this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND));
}
public PdbFile getFile() {
return pdbFile;
}
public void write(final Entry entry) throws WriteException {
write(entry.getDate(), entry.getValue());
}
void write(final Date time, final long value) throws WriteException {
final long timeValue = time.getTime();
final long adjustedValue = timeValue - pdbFile.getDateOffset();
assertValueInRange(adjustedValue);
assertValueInRange(value);
write(adjustedValue);
write(value);
}
private void assertValueInRange(final long value) {
if (value < 0) {
throw new IllegalArgumentException("value must not be negative: " + value);
}
if (value > MAX_VALUE) {
throw new IllegalArgumentException("max value is " + MAX_VALUE + " value was: " + value);
}
}
// visible for test
void write(final long value) throws WriteException {
assertValueInRange(value);
try {
outputStream.write(BitFiddling.long3(value));
outputStream.write(BitFiddling.long2(value));
outputStream.write(BitFiddling.long1(value));
outputStream.write(BitFiddling.long0(value));
} catch (final IOException e) {
throw new WriteException(e);
}
}
@Override
public void close() throws IOException {
outputStream.flush();
outputStream.close();
}
}

View File

@@ -0,0 +1,115 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import liquibase.exception.LiquibaseException;
public class PerformanceDb implements AutoCloseable {
private static final Logger LOGGER = Logger.getLogger(PerformanceDb.class.getCanonicalName());
private final TagsToFile tagsToFile;
public PerformanceDb(final Path dataDirectory) throws LiquibaseException {
tagsToFile = new TagsToFile(dataDirectory);
}
public void put(final Date date, final long value, final Tags tags) throws WriteException {
put(new Entry(date, value), tags);
}
public void put(final Entry entry, final Tags tags) throws WriteException {
put(Arrays.asList(entry), tags);
}
public void put(final Iterable<Entry> entries, final Tags tags) throws WriteException {
final long start = System.nanoTime();
double timeSpendInWrite = 0.0;
long count = 0;
PdbWriter writer = null;
PdbFile pdbFile = null;
try {
for (final Entry entry : entries) {
final Date date = entry.getDate();
final long value = entry.getValue();
if (pdbFile == null //
|| !pdbFile.getTimeRange().inRange(date)) // TODO @ahr
// correct
// would be
// to check
// if the
// date is
// in the
// available
// range
{
final long startWrite = System.nanoTime();
pdbFile = tagsToFile.getFile(date, tags);
timeSpendInWrite += (System.nanoTime() - startWrite) / 1_000_000.0;
}
if (writer == null || !writer.getFile().equals(pdbFile)) {
if (writer != null) {
writer.close();
}
writer = new PdbWriter(pdbFile);
}
writer.write(date, value);
count++;
}
} catch (final IOException e) {
throw new WriteException(e);
} finally {
if (writer != null) {
try {
writer.close();
} catch (final IOException e) {
throw new WriteException(e);
}
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite
+ " were spend in write");
}
}
public List<Entry> getAsList(final Tags tags) {
return get(tags).collect(Collectors.toList());
}
/**
* Return the entries as an unbound, ordered and non-parallel stream.
*
* @param tags
* @return {@link Stream} unbound, ordered and non-parallel
*/
public Stream<Entry> get(final Tags tags) {
final List<PdbFile> pdbFiles = tagsToFile.getFilesMatchingTags(tags);
final Iterator<Entry> iterator = new PdbFileIterator(pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
return StreamSupport.stream(spliterator, false);
}
@Override
public void close() throws Exception {
tagsToFile.close();
}
}

View File

@@ -0,0 +1,18 @@
package org.lucares.performance.db;
final class Query {
static String createQuery(final Tags tags) {
final StringBuilder result = new StringBuilder();
for (final String key : tags.getKeys()) {
tags.getValue(key).ifPresent(value -> {
result.append(Fields.prefixedKey(key));
result.append("=");
result.append(value);
result.append(" ");
});
}
return result.toString().trim();
}
}

View File

@@ -0,0 +1,18 @@
package org.lucares.performance.db;
public class ReadRuntimeException extends RuntimeException {
private static final long serialVersionUID = 1L;
public ReadRuntimeException(final String message, final Throwable cause) {
super(message, cause);
}
public ReadRuntimeException(final String message) {
super(message);
}
public ReadRuntimeException(final Throwable cause) {
super(cause);
}
}

View File

@@ -0,0 +1,11 @@
package org.lucares.performance.db;
import java.io.File;
import java.nio.file.Path;
public class StorageUtils {
static File createStorageFile(final Path parent, final Day day, final String name) {
return new File(parent.toFile(), day.format(File.separator) + File.separator + name);
}
}

View File

@@ -0,0 +1,108 @@
package org.lucares.performance.db;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.function.BiConsumer;
public class Tags {
private final Map<String, String> tags = new HashMap<>();
public Tags() {
super();
}
public Tags(final String key, final String value) {
super();
tags.put(key, value);
}
public Tags(final String key1, final String value1, final String key2, final String value2) {
super();
tags.put(key1, value1);
tags.put(key2, value2);
}
public Tags(final String key1, final String value1, final String key2, final String value2, final String key3,
final String value3) {
super();
tags.put(key1, value1);
tags.put(key2, value2);
tags.put(key3, value3);
}
public void put(final String key, final String value) {
tags.put(key, value);
}
public Optional<String> getValue(final String key) {
final String value = tags.get(key);
return Optional.ofNullable(value);
}
public Set<String> getKeys() {
return new TreeSet<>(tags.keySet());
}
public void forEach(final BiConsumer<String, String> keyValueConsumer) {
for (final Map.Entry<String, String> e : tags.entrySet()) {
keyValueConsumer.accept(e.getKey(), e.getValue());
}
}
@Override
public String toString() {
return String.valueOf(tags);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Tags other = (Tags) obj;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
public String abbreviatedRepresentation() {
final StringBuilder result = new StringBuilder();
final int maxLength = 200;
final SortedSet<String> keys = new TreeSet<>(tags.keySet());
final int cutAt = maxLength / (keys.size() * 2 + 2);
for (final String key : keys) {
result.append(substr(key, cutAt));
result.append("-");
result.append(substr(tags.get(key), cutAt));
result.append("_");
}
return substr(result.toString(), maxLength);
}
private static String substr(final String s, final int maxLength) {
return s.substring(0, Math.min(maxLength, s.length()));
}
}

View File

@@ -0,0 +1,191 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.lucares.ludb.Document;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldExistsException;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
import liquibase.exception.LiquibaseException;
public class TagsToFile implements AutoCloseable, CollectionUtils {
private final H2DB db;
private final Path dataDirectory;
public TagsToFile(final Path dataDirectory) {
super();
this.dataDirectory = dataDirectory;
try {
db = new H2DB(new File(dataDirectory.toFile(), "lu.db"));
try {
db.createField(Fields.DATE_OFFSET, FieldType.BIGINT);
} catch (final FieldExistsException e) {
// TODO @ahr ludb needs a hasField method, or a
// createIfNotExists
}
} catch (final LiquibaseException e) {
throw new RuntimeException(e);
}
}
private List<PdbFile> getFilesMatchingTagsExactly(final Tags tags) {
final List<PdbFile> files = getFilesMatchingTags(tags);
return filter(files, f -> f.getTags().equals(tags));
}
List<PdbFile> getFilesMatchingTags(final Tags tags) {
final List<PdbFile> result = new ArrayList<>();
final String query = Query.createQuery(tags);
try {
final List<Document> searchResult = db.search(query);
for (final Document document : searchResult) {
final PdbFile pdbFile = toPdbFile(document);
result.add(pdbFile);
}
} catch (final NullPointerException e) {
// TODO @ahr ludb should handle unknown fields better
}
Collections.sort(result, new PdbFileByTimeAsc());
return result;
}
private PdbFile toPdbFile(final Document document) {
final File file = document.getFile();
final long dateOffset = document.getPropertyLong(Fields.DATE_OFFSET);
final Tags tagsOfFile = toTags(document);
final PdbFile pdbFile = new PdbFile(dateOffset, file, tagsOfFile);
return pdbFile;
}
private Tags toTags(final Document document) {
final Tags tagsOfFile = new Tags();
for (final String key : document.getProperties().keySet()) {
if (Fields.isPrefixedKey(key)) {
final String value = document.getPropertyString(key);
tagsOfFile.put(Fields.stripPrefix(key), value);
}
}
return tagsOfFile;
}
public PdbFile getFile(final Date date, final Tags tags) throws FileNotFoundException, IOException {
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
final List<PdbFile> preResult = new ArrayList<>();
PdbFile result;
for (final PdbFile pdbFile : pdbFiles) {
final boolean inRange = pdbFile.getTimeRange().inRange(date);
if (inRange) {
final TimeRange availableTimeRange = getAvailableTimeRange(pdbFile);
if (availableTimeRange.inRange(date)) {
preResult.add(pdbFile);
}
}
}
if (preResult.isEmpty()) {
result = createNewPdbFile(date, tags);
} else {
if (preResult.size() != 1) {
throw new IllegalStateException("found not exactly one matching file: " + preResult);
}
result = preResult.get(0);
}
return result;
}
private TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
try (PdbReader reader = new PdbReader(pdbFile)) {
if (reader.canSeekTail(2)) {
reader.seekTail(2);
final Date lastDate = reader.readDate();
return new TimeRange(lastDate, pdbFile.getTimeRange().getTo());
} else {
return pdbFile.getTimeRange();
}
}
}
private PdbFile createNewPdbFile(final Date date, final Tags tags) {
final File file;
final long dateOffset;
PdbFile result;
file = createNewFile(date, tags);
dateOffset = DateUtils.getDateOffset(date);
db.addDocument(file);
ensureFieldsExist(tags);
tags.forEach((key, value) -> {
db.setProperty(file, Fields.prefixedKey(key), value);
});
db.setProperty(file, Fields.DATE_OFFSET, dateOffset);
result = new PdbFile(dateOffset, file, tags);
return result;
}
private void ensureFieldsExist(final Tags tags) {
final List<Field> fields = db.getAvailableFields();
final Map<String, Field> fieldsMap = toMap(fields, Field::getName);
tags.forEach((key, value) -> {
final String prefixedKey = Fields.prefixedKey(key);
final Field field = fieldsMap.get(prefixedKey);
if (field == null) {
db.createField(prefixedKey, FieldType.STRING);
}
});
}
private File createNewFile(final Date date, final Tags tags) {
final Day day = DateUtils.getDay(date);
final String name = tags.abbreviatedRepresentation() + UUID.randomUUID().toString();
final File result = StorageUtils.createStorageFile(dataDirectory, day, name);
try {
Files.createDirectories(result.getParentFile().toPath());
Files.createFile(result.toPath());
} catch (final IOException e) {
throw new IllegalStateException(e); // very unlikely
}
return result;
}
@Override
public void close() throws Exception {
db.close();
}
}

View File

@@ -0,0 +1,56 @@
package org.lucares.performance.db;
import java.util.Date;
import java.util.concurrent.TimeUnit;
public class TimeRange {
private final Date from;
private final Date to;
public TimeRange(final Date from, final Date to) {
if (!from.before(to)) {
throw new IllegalArgumentException("from date must be before to date. from: " + from + " to: " + to);
}
this.from = from;
this.to = to;
}
public Date getFrom() {
return from;
}
public Date getTo() {
return to;
}
public long length(final TimeUnit timeUnit) {
final long duration = to.getTime() - from.getTime();
return timeUnit.convert(duration, TimeUnit.MILLISECONDS);
}
public boolean inRange(final Date date) {
return from.compareTo(date) <= 0 && to.compareTo(date) >= 0;
}
public boolean intersect(final TimeRange timeRange) {
return inRange(timeRange.from) //
|| inRange(timeRange.to) //
|| timeRange.inRange(from)//
|| timeRange.inRange(to);
}
@Override
public String toString() {
return "[" + from + ":" + to + "]";
}
public static TimeRange today() {
final Date now = new Date();
final Date from = DateUtils.getMidnightSameDay(now);
final Date to = DateUtils.getMidnightNextDay(now);
return new TimeRange(from, to);
}
}

View File

@@ -0,0 +1,15 @@
package org.lucares.performance.db;
public class WriteException extends RuntimeException {
private static final long serialVersionUID = 1L;
public WriteException(final String message, final Throwable cause) {
super(message, cause);
}
public WriteException(final Throwable cause) {
super(cause);
}
}

View File

@@ -0,0 +1,115 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Date;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
public class PdbReaderWriterTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
@DataProvider(name = "providerWriteRead")
public Object[][] providerWriteRead() {
return new Object[][] { //
{ 1 }, //
{ 6 }, //
{ 0xffffffffL },//
};
}
@Test(dataProvider = "providerWriteRead")
public void testWriteRead(final long value) throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = PdbFile.today(file, new Tags());
final Date now = new Date(); // TODO @ahr might fail at midnight
final Entry entry = new Entry(now, value);
try (PdbWriter writer = new PdbWriter(pdbFile)) {
writer.write(entry);
}
try (final PdbReader reader = new PdbReader(pdbFile)) {
final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actual, entry);
}
}
public void testSeekTail() throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = PdbFile.today(file, new Tags());
try (PdbWriter writer = new PdbWriter(pdbFile)) {
writer.write(1);
writer.write(2);
writer.write(3);
writer.write(4);
writer.write(5);
}
try (final PdbReader reader = new PdbReader(pdbFile)) {
reader.seekTail(2);
final long four = reader.read();
final long five = reader.read();
Assert.assertEquals(four, 4, "second last value");
Assert.assertEquals(five, 5, "last value");
final long eof = reader.read();
Assert.assertEquals(eof, -1, "end of file");
}
}
public void testSeek() throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final PdbFile pdbFile = PdbFile.today(file, new Tags());
try (PdbWriter writer = new PdbWriter(pdbFile)) {
writer.write(1);
writer.write(2);
writer.write(3);
writer.write(4);
writer.write(5);
}
try (final PdbReader reader = new PdbReader(pdbFile)) {
reader.seek(2);
final long three = reader.read();
final long four = reader.read();
final long five = reader.read();
Assert.assertEquals(three, 3, "third value");
Assert.assertEquals(four, 4, "fourth value");
Assert.assertEquals(five, 5, "fifth value");
reader.seek(0);
final long first = reader.read();
Assert.assertEquals(first, 1, "first value");
}
}
}

View File

@@ -0,0 +1,162 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class PerformanceDbTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testInsertRead() throws Exception {
try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
final Date date = new Date();
final long value = 1;
final Tags tags = new Tags("myKey", "myValue");
performanceDb.put(date, value, tags);
final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
Assert.assertEquals(stream.size(), 1);
Assert.assertEquals(stream.get(0), new Entry(date, value));
}
}
public void testInsertIntoMultipleFilesRead() throws Exception {
try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
final Date dayOne = DateUtils.getDate(2016, 11, 1, 10, 0, 0);
final Date dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56);
final long valueOne = 1;
final long valueTwo = 2;
final Tags tags = new Tags("myKey", "myValue");
performanceDb.put(dayOne, valueOne, tags);
performanceDb.put(dayTwo, valueTwo, tags);
final List<Entry> stream = performanceDb.get(tags).collect(Collectors.toList());
Assert.assertEquals(stream.size(), 2);
Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne));
Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo));
}
}
private List<Entry> generateEntries(final TimeRange timeRange, final long n) {
final List<Entry> result = new ArrayList<>();
final long differenceInMs = timeRange.length(TimeUnit.MILLISECONDS) / n;
long currentTime = timeRange.getFrom().getTime();
for (long i = 0; i < n; i++) {
final Date date = new Date(currentTime);
final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE);
result.add(new Entry(date, value));
currentTime += differenceInMs;
}
return result;
}
public void testAppendToExistingFile() throws Exception {
try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
final TimeRange timeRange = TimeRange.today();
final long numberOfEntries = 2;
final Tags tags = new Tags("myKey", "one");
final List<Entry> entries = generateEntries(timeRange, numberOfEntries);
printEntries(entries, "");
for (final Entry entry : entries) {
performanceDb.put(entry, tags);
}
final List<Entry> actualEntries = performanceDb.getAsList(tags);
Assert.assertEquals(actualEntries, entries);
final File storageFileForToday = StorageUtils.createStorageFile(dataDirectory,
DateUtils.getDay(timeRange.getFrom()), "name doesn't matter");
final File storageFolderForToday = storageFileForToday.getParentFile();
final File[] filesInStorage = storageFolderForToday.listFiles();
Assert.assertEquals(filesInStorage.length, 1,
"one file in storage, but was: " + Arrays.asList(filesInStorage));
}
}
public void testInsertIntoMultipleFilesWithDifferentTags() throws Exception {
try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) {
final Date from = DateUtils.getDate(2016, 1, 1, 00, 00, 00);
final Date to = DateUtils.getDate(2016, 12, 31, 23, 59, 59);
final TimeRange timeRange = new TimeRange(from, to);
final long numberOfEntries = timeRange.length(TimeUnit.DAYS) * 2; // two
// entries
// per
// day
final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue");
final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries);
printEntries(entriesOne, "one");
performanceDb.put(entriesOne, tagsOne);
final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue");
final List<Entry> entriesTwo = generateEntries(timeRange, numberOfEntries);
printEntries(entriesTwo, "two");
performanceDb.put(entriesTwo, tagsTwo);
final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue");
final List<Entry> entriesThree = generateEntries(timeRange, numberOfEntries);
printEntries(entriesThree, "three");
performanceDb.put(entriesThree, tagsThree);
final List<Entry> actualEntriesOne = performanceDb.getAsList(tagsOne);
Assert.assertEquals(actualEntriesOne, entriesOne);
final List<Entry> actualEntriesTwo = performanceDb.getAsList(tagsTwo);
Assert.assertEquals(actualEntriesTwo, entriesTwo);
final List<Entry> actualEntriesThree = performanceDb.getAsList(tagsThree);
Assert.assertEquals(actualEntriesThree, entriesThree);
}
}
private void printEntries(final List<Entry> entriesOne, final String label) {
int index = 0;
for (final Entry entry : entriesOne) {
System.out.printf("%4d %s %d (%s)\n", index, DateUtils.format(entry.getDate()), entry.getValue(), label);
index++;
}
}
}

View File

@@ -0,0 +1,41 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Date;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class TagsToFilesTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
org.lucares.performance.db.FileUtils.delete(dataDirectory);
}
public void test() throws Exception {
try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) {
final Date date = new Date();
final Tags tags = new Tags("myKey", "myValue");
final PdbFile newFileForTags = tagsToFile.getFile(date, tags);
final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
Assert.assertEquals(newFileForTags, existingFileForTags);
}
}
}

View File

@@ -0,0 +1,37 @@
package org.lucares.performance.db;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
public class TimeRangeTest {
@DataProvider
Object[][] providerIntersect() {
final List<Object[]> result = new ArrayList<>();
final Date a = new Date(1000);
final Date b = new Date(2000);
final Date c = new Date(3000);
final Date d = new Date(4000);
result.add(new Object[] { new TimeRange(a, b), new TimeRange(c, d), false });
result.add(new Object[] { new TimeRange(a, c), new TimeRange(b, d), true });
result.add(new Object[] { new TimeRange(a, d), new TimeRange(b, d), true });
result.add(new Object[] { new TimeRange(a, d), new TimeRange(b, d), true });
result.add(new Object[] { new TimeRange(a, b), new TimeRange(b, d), true });
return result.toArray(new Object[result.size()][]);
}
@Test(dataProvider = "providerIntersect")
public void testIntersect(final TimeRange a, final TimeRange b, final boolean expected) throws Exception {
Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b);
Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b);
}
}