andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

replace the FolderStorage with DiskStorage

Browse Source 
- The DiskStorage uses only one file instead of millions.
  Also the block size is only 512 byte instead of 4kb, which
  helps to reduce the memory usage for short sequences.
- Update primitiveCollections to get the new LongList.range
  and LongList.rangeClosed methods.
- BSFile now stores Time&Value sequences and knows how to
  encode the time values with delta encoding.
- Doc had to do some magic tricks to save memory. The path
  was initialized lazy and stored as byte array. This is no
  longer necessary. The patch was replaced by the
  rootBlockNumber of the BSFile.
- Had to temporarily disable the 'in' queries.
- The stored values are now processed as stream of LongLists
  instead of Entry. The overhead for creating Entries is
  gone, so is the memory overhead, because Entry was an
  object and had a reference to the tags, which is
  unnecessary.
This commit is contained in:
Andreas Huber
2018-09-12 09:35:07 +02:00
parent 26dc052b95
commit 1182d76205
36 changed files with 799 additions and 1483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -1,18 +1,18 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
class PdbFile {
private final Tags tags;
private final Path path;
/**
* The rootBlockNumber to be used by {@link BSFile}
*/
private final long rootBlockNumber;
public PdbFile(final Path path, final Tags tags) {
this.path = path;
public PdbFile(final long rootBlockNumber, final Tags tags) {
this.rootBlockNumber = rootBlockNumber;
this.tags = tags;
}
@@ -20,20 +20,20 @@ class PdbFile {
return tags;
}
public Path getPath() {
return path;
public long getRootBlockNumber() {
return rootBlockNumber;
}
@Override
public String toString() {
return "PdbFile [" + path + " " + tags + "]\n";
return "PdbFile [tags=" + tags + ", rootBlockNumber=" + rootBlockNumber + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((path == null) ? 0 : path.hashCode());
result = prime * result + (int) (rootBlockNumber ^ (rootBlockNumber >>> 32));
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@@ -47,10 +47,7 @@ class PdbFile {
if (getClass() != obj.getClass())
return false;
final PdbFile other = (PdbFile) obj;
if (path == null) {
if (other.path != null)
return false;
} else if (!path.equals(other.path))
if (rootBlockNumber != other.rootBlockNumber)
return false;
if (tags == null) {
if (other.tags != null)
@@ -59,12 +56,4 @@ class PdbFile {
return false;
return true;
}
public boolean exists() throws ReadException {
try {
return Files.isRegularFile(path) && Files.size(path) >= ByteType.VersionByte.MIN_LENGTH;
} catch (final IOException e) {
throw new ReadException(e);
}
}
}

131
performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java
View File

@@ -1,131 +0,0 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayDeque;
import java.util.Collection;
import java.util.Iterator;
import java.util.Optional;
import java.util.Queue;
import java.util.function.Supplier;
import org.lucares.pdb.api.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class PdbFileIterator implements Iterator<Entry>, AutoCloseable {
private final static Logger LOGGER = LoggerFactory.getLogger(PdbFileIterator.class);
private static final class EntrySupplier implements Supplier<Entry>, AutoCloseable {
private final Queue<PdbFile> pdbFiles;
private PdbReader reader;
private PdbFile currentPdbFile;
private final Path storageBasePath;
public EntrySupplier(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
super();
this.storageBasePath = storageBasePath;
this.pdbFiles = new ArrayDeque<>(pdbFiles);
}
@Override
public Entry get() {
if (reader == null) {
nextFile();
}
if (reader == null) {
return null;
}
Entry entry = reader.readNullableEntry();
while (entry == null) {
nextFile();
if (reader == null) {
return null;
} else {
entry = reader.readEntry().orElse(null);
// A reader might return null, for a newly opened reader,
// if the file was created, but nothing has been written to
// disk yet.
// This might happen, because of buffering, or when an
// ingestion
// was cancelled.
}
}
return entry;
}
private void nextFile() {
if (reader != null) {
reader.close();
reader = null;
}
while (!pdbFiles.isEmpty()) {
currentPdbFile = pdbFiles.poll();
try {
if (Files.size(currentPdbFile.getPath()) > 0) {
reader = new PdbReader(storageBasePath, currentPdbFile);
break;
} else {
LOGGER.info("ignoring empty file " + currentPdbFile);
}
} catch (final FileNotFoundException e) {
LOGGER.warn("the pdbFile " + currentPdbFile.getPath() + " is missing", e);
} catch (final IOException e) {
throw new ReadException(e);
}
}
}
@Override
public void close() {
if (reader != null) {
reader.close();
}
}
}
private final EntrySupplier supplier;
private Optional<Entry> next = Optional.empty();
public PdbFileIterator(final Path storageBasePath, final Collection<PdbFile> pdbFiles) {
supplier = new EntrySupplier(storageBasePath, pdbFiles);
}
@Override
public boolean hasNext() {
final boolean result;
if (next.isPresent()) {
result = true;
} else {
next = Optional.ofNullable(supplier.get());
result = next.isPresent();
}
return result;
}
@Override
public Entry next() {
final Entry result = next.orElseGet(supplier::get);
next = Optional.empty();
return result;
}
@Override
public void close() {
supplier.close();
}
}

17
performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java
View File

@@ -1,17 +0,0 @@
package org.lucares.performance.db;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
class PdbFileUtils {
static OffsetDateTime dateOffset(final Path storageBasePath, final PdbFile pdbFile)
throws FileNotFoundException, IOException {
try (PdbReader reader = new PdbReader(storageBasePath, pdbFile)) {
reader.seekToLastValue();
return reader.getDateOffsetAtCurrentPosition();
}
}
}

72
performanceDb/src/main/java/org/lucares/performance/db/PdbFileViewer.java
View File

@@ -1,72 +0,0 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Path;
import org.lucares.pdb.api.Tags;
public class PdbFileViewer {
private static final Tags TAGS = Tags.create();
public static void main(final String[] args) throws FileNotFoundException, IOException {
final File file = new File(args[0]);
final Path baseDirectory = file.toPath().getParent();
final PdbFile pdbFile = new PdbFile(file.toPath().getFileName(), TAGS);
long countMeasurements = 0;
try (final PdbReader reader = new PdbReader(baseDirectory, pdbFile, false)) {
long value = 0;
int nextByte;
while ((nextByte = reader.readNextByte()) >= 0) {
final ByteType type = ByteType.getType(nextByte);
countMeasurements = countMeasurements + (type == ByteType.MEASUREMENT ? 1 : 0);
final long bytesValue = type.getValue(nextByte);
if (type == ByteType.CONTINUATION) {
value = value << ByteType.ContinuationByte.NUMBER_OF_VALUES_BITS;
value = value | type.getValue(nextByte);
} else {
value = bytesValue;
}
String additionalInfo = "";
if (ByteType.MEASUREMENT == ByteType.getType(reader.peekNextByte())) {
additionalInfo = format(value);
}
System.out.printf("%s %3d %3d %-14s %14d %s\n", toBinary(nextByte), nextByte, bytesValue, type, value,
additionalInfo);
}
}
System.out.println("Bytes: " + file.length());
System.out.println("Measurements: " + countMeasurements);
System.out.println("Bytes/Measurements: " + (file.length() / (double) countMeasurements));
}
private static String format(final long millis) {
final long years = millis / (1000L * 3600 * 24 * 365);
final long days = millis % (1000L * 3600 * 24 * 365) / (1000 * 3600 * 24);
final long hours = (millis % (1000 * 3600 * 24)) / (1000 * 3600);
final long minutes = (millis % (1000 * 3600)) / (1000 * 60);
final long seconds = (millis % (1000 * 60)) / 1000;
final long ms = millis % 1000;
if (years > 0) {
return String.format("%d years %d days %02d:%02d:%02d,%03d", years, days, hours, minutes, seconds, ms);
} else if (days > 0) {
return String.format("%d days %02d:%02d:%02d,%03d", days, hours, minutes, seconds, ms);
}
return String.format("%02d:%02d:%02d,%03d", hours, minutes, seconds, ms);
}
private static String toBinary(final int b) {
return String.format("%8s", Integer.toBinaryString(b)).replace(" ", "0");
}
}

180
performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java
View File

@@ -1,180 +0,0 @@
package org.lucares.performance.db;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.Optional;
import org.lucares.pdb.api.Entry;
class PdbReader implements AutoCloseable {
private static final int PEEK_NOT_SET = Integer.MIN_VALUE;
static final long VERSION = 1;
private final InputStream data;
private long dateOffsetAtCurrentLocation = 0;
private long index = 0;
private int peekedByte = PEEK_NOT_SET;
private final PdbFile pdbFile;
public PdbReader(final Path storageBasePath, final PdbFile pdbFile) throws ReadException {
this(storageBasePath, pdbFile, true);
}
PdbReader(final Path storageBasePath, final PdbFile pdbFile, final boolean initialize) throws ReadException {
super();
try {
this.pdbFile = pdbFile;
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.data = new BufferedInputStream(new FileInputStream(storageFile));
if (initialize) {
init();
}
} catch (final FileNotFoundException e) {
throw new ReadException(e);
}
}
private void init() {
try {
final long version = readValue(ByteType.VERSION);
if (version == -1) {
throw new IllegalStateException("Cannot read empty file. The file must have at least a version. "
+ "Otherwise we don't know in which version a writer might append data.");
} else if (version != VERSION) {
throw new IllegalStateException(
"The file is not of version " + VERSION + ". Actual version: " + version);
}
} catch (final IOException e) {
throw new ReadException(e);
}
}
public PdbFile getPdbFile() {
return pdbFile;
}
/**
* Seek to the end of the file.
*
* @throws ReadRuntimeException
* if an IOException occurs
*/
public void seekToLastValue() {
while (readEntry().isPresent()) {
// seek to the end
// TODO @ahr add date offsets every x kb, so we don't have
// to read the whole file
}
}
@Override
public void close() {
try {
data.close();
} catch (final IOException e) {
throw new ReadRuntimeException(e);
}
}
Entry readNullableEntry() throws ReadRuntimeException {
try {
final long epochMilliIncrement = readValue(ByteType.DATE_INCREMENT);
if (epochMilliIncrement < 0) {
return null;
}
final long epochMilli = dateOffsetAtCurrentLocation + epochMilliIncrement;
final long value = readValue(ByteType.MEASUREMENT);
if (value < 0) {
return null;
}
dateOffsetAtCurrentLocation = epochMilli;
return new Entry(epochMilli, value, pdbFile.getTags());
} catch (final IOException e) {
throw new ReadException(e);
}
}
public Optional<Entry> readEntry() throws ReadRuntimeException {
final Entry entry = readNullableEntry();
return Optional.ofNullable(entry);
}
public OffsetDateTime getDateOffsetAtCurrentPosition() {
return OffsetDateTime.ofInstant(Instant.ofEpochMilli(dateOffsetAtCurrentLocation), ZoneId.of("UTC"));
}
public long readValue(final ByteType byteType) throws IOException {
final long firstByteValueBits = byteType.getValueBits();
int firstByte = readNextByte();
if (!byteType.isValid(firstByte)) {
if (firstByte < 0) {
// end of file reached
return -1;
} else if (ByteType.DATE_OFFSET.isValid(firstByte)) {
final long dateOffsetInit = firstByte & ByteType.DATE_OFFSET.getValueBits();
this.dateOffsetAtCurrentLocation = readContinuationBytes(dateOffsetInit);
firstByte = readNextByte();
} else {
throw new FileCorruptException(
"File corrupt at " + index + ". Byte type was " + ByteType.getType(firstByte));
}
}
final long value = firstByte & firstByteValueBits;
return readContinuationBytes(value);
}
int readNextByte() throws IOException {
final int result;
if (peekedByte == PEEK_NOT_SET) {
result = data.read();
} else {
result = peekedByte;
peekedByte = PEEK_NOT_SET;
}
index++;
return result;
}
int peekNextByte() throws IOException {
if (peekedByte == PEEK_NOT_SET) {
peekedByte = data.read();
}
return peekedByte;
}
private long readContinuationBytes(long value) throws IOException {
int nextByte;
while ((nextByte = peekNextByte()) >= 0 && ByteType.CONTINUATION.isValid(nextByte)) {
value = value << ByteType.ContinuationByte.NUMBER_OF_VALUES_BITS;
value = value | (nextByte & ByteType.CONTINUATION.getValueBits());
readNextByte();
}
return value;
}
}

128
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -1,100 +1,33 @@
package org.lucares.performance.db;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.Flushable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.Optional;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.diskstorage.DiskStorage;
/**
* File format description:
* <p>
* We store non-negative long values for epoch milli and a measurement (usually
* duration in ms). Both values are stored as pairs, so that we get
* date-measurement-date-measurement-date... . The date values are stored as
* difference to the previous date. Every few kilobytes we add an absolute
* offset, so that we can synchronize and don't have to read the whole file when
* we want to append.
*
* <p>
* For example we want to store the following values:
*
* <pre>
* 2009-02-45T12:31:30.30+0100 123 the date is 1234567890 in epoch millis
* 2009-02-45T01:06:39.39+0100 456 the date is 1234569999 in epoch millis
* </pre>
* <p>
* We would first store the offset 1234567890, then die first pair. The date is
* stored as the offset to the last value (which was the offset), so it is 0.
* Then we store the measurement. Next we store the second pair. The date
* difference is 2109 and the measurement is 456.
* <p>
* Each value is stored with a variable length byte sequence. The idea is
* similar to the encoding of UTF-8. But we differentiate between several
* different types of values.
* <ol>
* <li>version, start with 000001
* <li>number of entries up until this point in this file, 00001
* <li>date offsets with absolute values for epoch milli, start with 0001
* <li>date increments to the previous date value, start with 01
* <li>measurements, start with 001
* <li>continuation bytes, start with 1
* </ol>
*
* This is different from UTF-8. We do not encode the number of continuation
* bytes. Therefore we loose UTF-8's self validation feature and we cannot skip
* to the next value without reading all continuation bytes. But it is a little
* bit more efficient, because each continuation byte can store 7 bit instead of
* 6. A four byte sequence in UTF-8 can store 21 bits whereas a four byte
* sequence in this scheme stores 27 bits for values and 26 bits for date
* increments. But it is not as efficent for one byte sequences. On the other
* hand we also encode five different value types.
* <p>
* The encoding looks as follows:
* <p>
* The first byte starts with 00001 for meta-data. The three remaining bits are
* used for the version number. 001 in our case. So the first byte looks like
* this. 00001001
* <p>
* The second byte starts with 0001 for date offsets, 01 for date increments and
* 001 for measurements. All continuation bytes start with 1. E.g. The
* measurement 202 has the unsigned bit representation 11001010. The first byte
* of a measurement value starts with 001, so we have room for the first 5 bits.
* But we need 8 bits. So we must add another byte. The second byte starts with
* 1 and has room for 7 bits. The result looks like this: <b>001</b><i>00001<i>
* <b>1</b><i>1001010</i>
*/
class PdbWriter implements AutoCloseable, Flushable {
private static final boolean APPEND = true;
private static final int MAX_BYTES_PER_VALUE = 10;
private final byte[] buffer = new byte[MAX_BYTES_PER_VALUE];
private final OutputStream outputStream;
private final PdbFile pdbFile;
private long lastEpochMilli;
PdbWriter(final Path storageBasePath, final PdbFile pdbFile) throws IOException {
private final BSFile bsFile;
PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) throws IOException {
this.pdbFile = pdbFile;
final File storageFile = storageBasePath.resolve(pdbFile.getPath()).toFile();
this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
if (storageFile.exists() && storageFile.length() > 0) {
// TODO @ahr check version
bsFile = BSFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
final Optional<Long> optionalLastValue = bsFile.getLastValue();
final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(storageBasePath, pdbFile);
lastEpochMilli = dateOffset.toInstant().toEpochMilli();
if (optionalLastValue.isPresent()) {
lastEpochMilli = optionalLastValue.get();
} else {
writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream);
writeValue(0, ByteType.DATE_OFFSET, outputStream);
lastEpochMilli = 0;
}
}
@@ -123,8 +56,8 @@ class PdbWriter implements AutoCloseable, Flushable {
assertValueInRange(epochMilliIncrement);
assertValueInRange(value);
writeValue(epochMilliIncrement, ByteType.DATE_INCREMENT, outputStream);
writeValue(value, ByteType.MEASUREMENT, outputStream);
bsFile.appendTimeValue(epochMilli, value);
lastEpochMilli = epochMilli;
} catch (final IOException e) {
throw new WriteException(e);
@@ -138,49 +71,24 @@ class PdbWriter implements AutoCloseable, Flushable {
}
@Override
public void close() throws IOException {
outputStream.flush();
outputStream.close();
public void close() {
bsFile.close();
}
@Override
public void flush() throws IOException {
outputStream.flush();
public void flush() {
bsFile.flush();
}
public void writeValue(final long value, final ByteType byteSequenceType, final OutputStream output)
public static void writeEntry(final PdbFile pdbFile, final DiskStorage diskStorage, final Entry... entries)
throws IOException {
int index = buffer.length - 1;
final long maxFirstByteValue = byteSequenceType.getFirstByteMaxValue();
long val = value;
while (val > maxFirstByteValue) {
// handles continuation bytes
buffer[index] = (byte) ((val & ByteType.CONTINUATION.getValueBits())
| ByteType.CONTINUATION.getBytePrefix());
index--;
val = val >> ByteType.ContinuationByte.NUMBER_OF_VALUES_BITS;
}
buffer[index] = (byte) (val | byteSequenceType.getBytePrefix());
output.write(buffer, index, buffer.length - index);
}
public static void writeEntry(final Path storageBasePath, final PdbFile pdbFile, final Entry... entries)
throws IOException {
try (PdbWriter writer = new PdbWriter(storageBasePath, pdbFile)) {
try (PdbWriter writer = new PdbWriter(pdbFile, diskStorage)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
}
public static void init(final Path storageBasePath, final PdbFile result) throws IOException {
writeEntry(storageBasePath, result);
}
@Override
public String toString() {
return "PdbWriter [pdbFile=" + pdbFile + ", lastEpochMilli=" + lastEpochMilli + "]";

24
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -9,14 +9,12 @@ import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.SortedSet;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
@@ -143,8 +141,7 @@ public class PerformanceDb implements AutoCloseable {
* Return the entries as an unbound, ordered and non-parallel stream.
*
* @param query
* @param groupBy
* the tag to group by
* @param groupBy the tag to group by
* @return {@link Result}
*/
public Result get(final String query, final List<String> groupBy) {
@@ -162,7 +159,7 @@ public class PerformanceDb implements AutoCloseable {
private Result toResult(final Grouping grouping) {
final List<GroupResult> groupResults = new ArrayList<>();
for (final Group group : grouping.getGroups()) {
final Stream<Entry> stream = toStream(group.getFiles());
final Stream<LongList> stream = TimeValueStreamFactory.toStream(group.getFiles(), db.getDiskStorage());
final GroupResult groupResult = new GroupResult(stream, group.getTags());
groupResults.add(groupResult);
}
@@ -170,21 +167,6 @@ public class PerformanceDb implements AutoCloseable {
return result;
}
private Stream<Entry> toStream(final List<PdbFile> pdbFiles) {
final PdbFileIterator iterator = new PdbFileIterator(db.getStorageBasePath(), pdbFiles);
final Spliterator<Entry> spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED);
final Stream<Entry> stream = StreamSupport.stream(spliterator, false);
final Stream<Entry> result = stream.onClose(() -> {
try {
iterator.close();
} catch (final RuntimeException e) {
LOGGER.info("runtime exception while closing iterator", e);
}
});
return result;
}
@Override
public void close() {
tagsToFile.close();

30
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,7 +1,6 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -15,7 +14,6 @@ import java.util.function.Consumer;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.FolderStoragePathResolver;
import org.lucares.pdb.datastore.PdbDB;
import org.lucares.utils.CollectionUtils;
import org.slf4j.Logger;
@@ -38,7 +36,7 @@ public class TagsToFile implements AutoCloseable {
writers.add(writer);
}
public Optional<PdbWriter> writer(final PdbFile pdbFile) {
public Optional<PdbWriter> findWriterForPdbFile(final PdbFile pdbFile) {
return writers.stream().filter(w -> Objects.equals(w.getPdbFile(), pdbFile)).findAny();
}
}
@@ -68,13 +66,12 @@ public class TagsToFile implements AutoCloseable {
}
private List<PdbFile> toPdbFiles(final List<Doc> searchResult) {
final List<PdbFile> result = new ArrayList<>();
final List<PdbFile> result = new ArrayList<>(searchResult.size());
for (final Doc document : searchResult) {
final FolderStoragePathResolver resolver = db.getFolderStoragePathResolver();
final Path path = document.getAbsolutePath(resolver);
final long rootBlockNumber = document.getRootBlockNumber();
final Tags tags = document.getTags();
final PdbFile pdbFile = new PdbFile(path, tags);
final PdbFile pdbFile = new PdbFile(rootBlockNumber, tags);
result.add(pdbFile);
}
@@ -94,8 +91,8 @@ public class TagsToFile implements AutoCloseable {
} else {
final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
pdbFiles.removeIf(f -> !f.exists());
final List<Optional<PdbWriter>> optionalWriters = CollectionUtils.map(pdbFiles, writersForTags::writer);
final List<Optional<PdbWriter>> optionalWriters = CollectionUtils.map(pdbFiles,
writersForTags::findWriterForPdbFile);
final List<Optional<PdbWriter>> existingWriters = CollectionUtils.filter(optionalWriters,
Optional::isPresent);
final List<PdbWriter> writers = CollectionUtils.map(existingWriters, Optional::get);
@@ -143,11 +140,11 @@ public class TagsToFile implements AutoCloseable {
final WriterCache writerCache = entry.getValue();
for (final PdbWriter writer : writerCache.getWriters()) {
LOGGER.trace("closing cached writer: {}", writer.getPdbFile().getPath());
LOGGER.trace("closing cached writer: {}", writer.getPdbFile());
try {
writer.close();
} catch (final RuntimeException | IOException e) {
} catch (final RuntimeException e) {
LOGGER.warn("failed to close writer: " + writer.getPdbFile(), e);
}
}
@@ -160,7 +157,7 @@ public class TagsToFile implements AutoCloseable {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(db.getStorageBasePath(), pdbFile);
final PdbWriter result = new PdbWriter(pdbFile, db.getDiskStorage());
getOrInit(tags).addWriter(result);
@@ -176,10 +173,9 @@ public class TagsToFile implements AutoCloseable {
private PdbFile createNewPdbFile(final Tags tags) throws IOException {
final Path storageFile = db.createNewFile(tags);
final long rootBlockNumber = db.createNewFile(tags);
final PdbFile result = new PdbFile(storageFile, tags);
PdbWriter.init(db.getStorageBasePath(), result);
final PdbFile result = new PdbFile(rootBlockNumber, tags);
return result;
}
@@ -190,7 +186,7 @@ public class TagsToFile implements AutoCloseable {
try {
consumer.accept(writer);
} catch (final RuntimeException e) {
LOGGER.warn("failed to close writer for file " + writer.getPdbFile().getPath(), e);
LOGGER.warn("failed to close writer for file " + writer.getPdbFile(), e);
}
}
}
@@ -215,7 +211,7 @@ public class TagsToFile implements AutoCloseable {
try {
LOGGER.trace("flushing writer {}", t.getPdbFile());
t.flush();
} catch (final IOException e) {
} catch (final RuntimeException e) {
throw new WriteException(e);
}
});

63
performanceDb/src/main/java/org/lucares/performance/db/TimeValueStreamFactory.java Normal file
View File

@@ -0,0 +1,63 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Stream;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.RuntimeIOException;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.diskstorage.DiskStorage;
public class TimeValueStreamFactory {
private static class TimeStampDifferencesDecoder implements Function<LongList, LongList> {
// TODO move the timestamp correction into the BSFile
@Override
public LongList apply(final LongList t) {
long lastTimeValue = 0;
for (int i = 0; i < t.size(); i += 2) {
lastTimeValue += t.get(i);
t.set(i, lastTimeValue);
}
return t;
}
}
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {
private final DiskStorage diskStorage;
public PdbFileToLongStream(final DiskStorage diskStorage) {
this.diskStorage = diskStorage;
}
@Override
public Stream<LongList> apply(final PdbFile pdbFile) {
try {
final BSFile bsFile = BSFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);
// time values (every second value) is stored as difference to the previous
// value
// the other values are measurements and are stored with their real value
final Stream<LongList> result = bsFile.streamOfLongLists().map(new TimeStampDifferencesDecoder());
return result;
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
}
}
public static Stream<LongList> toStream(final List<PdbFile> pdbFiles, final DiskStorage diskStorage) {
final Stream<LongList> longStream = pdbFiles.stream().flatMap(new PdbFileToLongStream(diskStorage));
return longStream;
}
}

258
performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java
View File

@@ -1,129 +1,129 @@
package org.lucares.performance.db;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
public class PdbReaderWriterTest {
private Path dataDirectory;
private static final Tags TAGS = Tags.create();
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
org.lucares.utils.file.FileUtils.delete(dataDirectory);
}
@DataProvider(name = "providerWriteRead")
public Iterator<Object[]> providerWriteRead() {
final OffsetDateTime two_sixteen = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L, 202L, 255L, 256L, 8191L, 8192L, 1048575L,
1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L, 2199023255552L, 281474976710655L,
281474976710656L, 36028797018963967L, 36028797018963968L, 4611686018427387901L, 4611686018427387904L);
final List<Object[]> result = new ArrayList<>();
// single values
for (final Long value : values) {
result.add(new Object[] { Arrays.asList(new Entry(two_sixteen, value, TAGS)) });
}
// multivalues
final List<Entry> entries = new ArrayList<>();
for (int i = 0; i < 100; i++) {
final long epochMilli = 123456 * i;
final OffsetDateTime date = OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC"));
entries.add(new Entry(date, i, TAGS));
}
result.add(new Object[] { entries });
return result.iterator();
}
@Test(dataProvider = "providerWriteRead")
public void testWriteRead(final List<Entry> entries) throws Exception {
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
writer.write(entry);
}
}
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
for (final Entry entry : entries) {
final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actual, entry);
}
reader.readEntry().ifPresent(e -> {
throw new AssertionError();
});
}
}
@Test(expectedExceptions = FileCorruptException.class)
public void testReadExceptionOnCorruptEntries() throws Exception {
final Entry entryA = new Entry(1, 1, TAGS);
final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
final Path relativePath = dataDirectory.relativize(file.toPath());
final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
writer.write(entryA);
}
// make the file corrupt
// two date consecutive increments will never happen in valid data
final byte[] corruptEntries = new byte[] { //
ByteType.DATE_INCREMENT.getBytePrefixAsByte(), //
ByteType.DATE_INCREMENT.getBytePrefixAsByte() //
};
Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND);
try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError());
Assert.assertEquals(actualA, entryA);
reader.readEntry(); // should throw FileCorruptException
}
}
}
//package org.lucares.performance.db;
//
//import java.io.File;
//import java.io.IOException;
//import java.nio.file.Files;
//import java.nio.file.Path;
//import java.nio.file.StandardOpenOption;
//import java.time.Instant;
//import java.time.OffsetDateTime;
//import java.time.ZoneId;
//import java.util.ArrayList;
//import java.util.Arrays;
//import java.util.Iterator;
//import java.util.List;
//
//import org.lucares.pdb.api.Entry;
//import org.lucares.pdb.api.Tags;
//import org.testng.Assert;
//import org.testng.annotations.AfterMethod;
//import org.testng.annotations.BeforeMethod;
//import org.testng.annotations.DataProvider;
//import org.testng.annotations.Test;
//
//@Test
//public class PdbReaderWriterTest {
//
// private Path dataDirectory;
//
// private static final Tags TAGS = Tags.create();
//
// @BeforeMethod
// public void beforeMethod() throws IOException {
// dataDirectory = Files.createTempDirectory("pdb");
// }
//
// @AfterMethod
// public void afterMethod() throws IOException {
// org.lucares.utils.file.FileUtils.delete(dataDirectory);
// }
//
// @DataProvider(name = "providerWriteRead")
// public Iterator<Object[]> providerWriteRead() {
//
// final OffsetDateTime two_sixteen = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
//
// final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L, 202L, 255L, 256L, 8191L, 8192L, 1048575L,
// 1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L, 2199023255552L, 281474976710655L,
// 281474976710656L, 36028797018963967L, 36028797018963968L, 4611686018427387901L, 4611686018427387904L);
//
// final List<Object[]> result = new ArrayList<>();
//
// // single values
// for (final Long value : values) {
// result.add(new Object[] { Arrays.asList(new Entry(two_sixteen, value, TAGS)) });
// }
//
// // multivalues
// final List<Entry> entries = new ArrayList<>();
// for (int i = 0; i < 100; i++) {
//
// final long epochMilli = 123456 * i;
//
// final OffsetDateTime date = OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC"));
//
// entries.add(new Entry(date, i, TAGS));
// }
// result.add(new Object[] { entries });
//
// return result.iterator();
// }
//
// @Test(dataProvider = "providerWriteRead")
// public void testWriteRead(final List<Entry> entries) throws Exception {
//
// final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
// final Path relativePath = dataDirectory.relativize(file.toPath());
// final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
//
// try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
// for (final Entry entry : entries) {
// writer.write(entry);
// }
// }
//
// try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
//
// for (final Entry entry : entries) {
//
// final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError());
//
// Assert.assertEquals(actual, entry);
// }
// reader.readEntry().ifPresent(e -> {
// throw new AssertionError();
// });
// }
// }
//
// @Test(expectedExceptions = FileCorruptException.class)
// public void testReadExceptionOnCorruptEntries() throws Exception {
//
// final Entry entryA = new Entry(1, 1, TAGS);
//
// final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
// final Path relativePath = dataDirectory.relativize(file.toPath());
// final PdbFile pdbFile = new PdbFile(relativePath, TAGS);
//
// try (PdbWriter writer = new PdbWriter(dataDirectory, pdbFile)) {
// writer.write(entryA);
// }
//
// // make the file corrupt
// // two date consecutive increments will never happen in valid data
// final byte[] corruptEntries = new byte[] { //
// ByteType.DATE_INCREMENT.getBytePrefixAsByte(), //
// ByteType.DATE_INCREMENT.getBytePrefixAsByte() //
// };
//
// Files.write(file.toPath(), corruptEntries, StandardOpenOption.APPEND);
//
// try (final PdbReader reader = new PdbReader(dataDirectory, pdbFile)) {
//
// final Entry actualA = reader.readEntry().orElseThrow(() -> new AssertionError());
// Assert.assertEquals(actualA, entryA);
//
// reader.readEntry(); // should throw FileCorruptException
// }
// }
//}

179
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -10,15 +10,15 @@ import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.collections4.CollectionUtils;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
@@ -45,11 +45,12 @@ public class PerformanceDbTest {
db.put(new Entry(date, value, tags));
final Result result = db.get(Query.createQuery(tags));
final List<Entry> stream = result.singleGroup().asList();
final LongList stream = result.singleGroup().flatMap();
Assert.assertEquals(stream.size(), 1);
Assert.assertEquals(stream.size(), 2);
Assert.assertEquals(stream.get(0), new Entry(date, value, tags));
Assert.assertEquals(stream.get(0), date.toInstant().toEpochMilli());
Assert.assertEquals(stream.get(1), value);
}
}
@@ -65,12 +66,14 @@ public class PerformanceDbTest {
db.put(new Entry(dayOne, valueOne, tags));
db.put(new Entry(dayTwo, valueTwo, tags));
final List<Entry> stream = db.get(Query.createQuery(tags)).singleGroup().asList();
final LongList stream = db.get(Query.createQuery(tags)).singleGroup().flatMap();
Assert.assertEquals(stream.size(), 2);
Assert.assertEquals(stream.size(), 4);
Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne, tags));
Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo, tags));
Assert.assertEquals(stream.get(0), dayOne.toInstant().toEpochMilli());
Assert.assertEquals(stream.get(1), valueOne);
Assert.assertEquals(stream.get(2), dayTwo.toInstant().toEpochMilli());
Assert.assertEquals(stream.get(3), valueTwo);
}
}
@@ -88,7 +91,17 @@ public class PerformanceDbTest {
return result;
}
public void testAppendToExistingFile() throws Exception {
@DataProvider
public Object[][] providerAppendToExistingFile() throws Exception {
return new Object[][] { //
{ 2 }, //
{ 100 }, //
{ 500 }, //
};
}
@Test(dataProvider = "providerAppendToExistingFile")
public void testAppendToExistingFile(final long numberOfEntries) throws Exception {
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
@@ -97,34 +110,73 @@ public class PerformanceDbTest {
final int day = 2;
final TimeRange timeRange = TimeRange.ofDay(DateUtils.getDate(year, month, day, 1, 1, 1));
final long numberOfEntries = 2;
final Tags tags = Tags.create("myKey", "one");
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0, tags);
printEntries(entries, "");
// printEntries(entries, "");
for (final Entry entry : entries) {
db.put(entry);
}
final List<Entry> actualEntries = db.get(Query.createQuery(tags)).singleGroup().asList();
Assert.assertEquals(actualEntries, entries);
final LongList actualEntries = db.get(Query.createQuery(tags)).singleGroup().flatMap();
Assert.assertEquals(actualEntries.size(), entries.size() * 2);
final Path storageBasePath = DataStore.storageDirectory(dataDirectory);
final List<Path> filesInStorage = FileUtils.listRecursively(storageBasePath);
for (int i = 0; i < entries.size(); i++) {
final Entry entry = entries.get(i);
final long epochMilli = entry.getEpochMilli();
final long value = entry.getValue();
Assert.assertEquals(filesInStorage.size(), 2, "the created file and the listing.csv");
final Path tagSpecificFile = filesInStorage.get(0);
final PdbFile pdbFile = new PdbFile(tagSpecificFile, tags);
try (PdbReader pdbReader = new PdbReader(storageBasePath, pdbFile)) {
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(0));
Assert.assertEquals(pdbReader.readEntry().get(), entries.get(1));
Assert.assertEquals(pdbReader.readEntry().isPresent(), false);
Assert.assertEquals(actualEntries.get(i * 2), epochMilli);
Assert.assertEquals(actualEntries.get(i * 2 + 1), value);
}
}
}
@DataProvider
public Object[][] providerAppendToExistingFileWithRestart() throws Exception {
return new Object[][] { //
{ 2 }, //
{ 100 }, //
{ 500 }, //
};
}
@Test(dataProvider = "providerAppendToExistingFileWithRestart")
public void testAppendToExistingFileWithRestart(final long numberOfEntries) throws Exception {
final Tags tags;
final List<Entry> expected = new ArrayList<>();
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final int year = 2016;
final int month = 1;
final int day = 2;
tags = Tags.create("myKey", "one");
final TimeRange timeRange = TimeRange.ofDay(DateUtils.getDate(year, month, day, 1, 1, 1));
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0, tags);
db.put(entries);
expected.addAll(entries);
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final int year = 2016;
final int month = 1;
final int day = 3;
final TimeRange timeRange = TimeRange.ofDay(DateUtils.getDate(year, month, day, 1, 1, 1));
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, 0, tags);
db.put(entries);
expected.addAll(entries);
final LongList actualEntries = db.get(Query.createQuery(tags)).singleGroup().flatMap();
Assert.assertEquals(actualEntries.size(), expected.size() * 2);
Assert.assertEquals(actualEntries, toExpectedValues(expected));
}
}
@@ -139,7 +191,8 @@ public class PerformanceDbTest {
final Tags tagsCommon = Tags.create("commonKey", "commonValue");
final Tags tagsOne = Tags.create("myKey", "one", "commonKey", "commonValue");
final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final List<Entry> entriesOne = generateEntries(timeRange, numberOfEntries, 1, tagsOne);
db.put(entriesOne);
printEntries(entriesOne, "one");
final Tags tagsTwo = Tags.create("myKey", "two", "commonKey", "commonValue");
@@ -152,22 +205,25 @@ public class PerformanceDbTest {
printEntries(entriesThree, "three");
db.put(entriesThree);
final List<Entry> actualEntriesOne = db.get(Query.createQuery(tagsOne)).singleGroup().asList();
Assert.assertEquals(actualEntriesOne, entriesOne);
final LongList actualEntriesOne = db.get(Query.createQuery(tagsOne)).singleGroup().flatMap();
Assert.assertEquals(actualEntriesOne, toExpectedValues(entriesOne));
final List<Entry> actualEntriesTwo = db.get(Query.createQuery(tagsTwo)).singleGroup().asList();
Assert.assertEquals(actualEntriesTwo, entriesTwo);
final LongList actualEntriesTwo = db.get(Query.createQuery(tagsTwo)).singleGroup().flatMap();
Assert.assertEquals(actualEntriesTwo, toExpectedValues(entriesTwo));
final List<Entry> actualEntriesThree = db.get(Query.createQuery(tagsThree)).singleGroup().asList();
Assert.assertEquals(actualEntriesThree, entriesThree);
final LongList actualEntriesThree = db.get(Query.createQuery(tagsThree)).singleGroup().flatMap();
Assert.assertEquals(actualEntriesThree, toExpectedValues(entriesThree));
final List<Entry> actualEntriesAll = db.get(Query.createQuery(tagsCommon)).singleGroup().asList();
final LongList actualEntriesAll = db.get(Query.createQuery(tagsCommon)).singleGroup().flatMap();
final List<Entry> expectedAll = CollectionUtils.collate(entriesOne,
CollectionUtils.collate(entriesTwo, entriesThree, EntryByDateComparator.INSTANCE),
EntryByDateComparator.INSTANCE);
final LongList expectedValues = toExpectedValues(expectedAll);
actualEntriesAll.sort(EntryByDateComparator.INSTANCE);
Assert.assertEquals(actualEntriesAll, expectedAll);
actualEntriesAll.sort();
expectedValues.sort();
Assert.assertEquals(actualEntriesAll, expectedValues);
}
}
@@ -183,9 +239,9 @@ public class PerformanceDbTest {
final Tags tagsOne = Tags.create(key, "one", "commonKey", "commonValue");
final Tags tagsTwo = Tags.create(key, "two", "commonKey", "commonValue");
final Tags tagsThree = Tags.create("commonKey", "commonValue");
final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwo, 2);
final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 3);
final LongList entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final LongList entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwo, 2);
final LongList entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 3);
final Result result = db.get("commonKey=commonValue", Arrays.asList(key));
@@ -195,12 +251,12 @@ public class PerformanceDbTest {
final Tags groupedBy = groupResult.getGroupedBy();
if (groupedBy.equals(Tags.create(key, "one"))) {
Assert.assertEquals(groupResult.asList(), entriesOne);
Assert.assertEquals(groupResult.flatMap(), entriesOne);
} else if (groupedBy.equals(Tags.create(key, "two"))) {
Assert.assertEquals(groupResult.asList(), entriesTwo);
Assert.assertEquals(groupResult.flatMap(), entriesTwo);
} else if (groupedBy.isEmpty()) {
Assert.assertEquals(groupResult.asList(), entriesThree);
Assert.assertEquals(groupResult.flatMap(), entriesThree);
} else {
Assert.fail("unexpected group: " + groupResult.getGroupedBy());
}
@@ -223,10 +279,10 @@ public class PerformanceDbTest {
final Tags tagsTwoB = Tags.create(key1, "two", key2, "bbb", "commonKey", "commonValue");
final Tags tagsThree = Tags.create(key1, "three", "commonKey", "commonValue");
final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwoA, 2);
final LongList entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final LongList entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwoA, 2);
entriesTwo.addAll(storeEntries(db, timeRange, numberOfEntries, tagsTwoB, 3));
final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 4);
final LongList entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 4);
final Result result = db.get("commonKey=commonValue", Arrays.asList(key1, key2));
@@ -236,19 +292,19 @@ public class PerformanceDbTest {
final Tags groupedBy = groupResult.getGroupedBy();
if (groupedBy.equals(Tags.create(key1, "one", key2, "aaa"))) {
Assert.assertEquals(groupResult.asList(), entriesOne);
Assert.assertEquals(groupResult.flatMap(), entriesOne);
} else if (groupedBy.equals(Tags.create(key1, "two", key2, "bbb"))) {
// there is no defined order of the entries.
// eventually we might return them in ascending order, but
// that is not yet implemented
final List<Entry> actualEntries = groupResult.asList();
final LongList actualEntries = groupResult.flatMap();
entriesTwo.sort(EntryByDateComparator.INSTANCE);
actualEntries.sort(EntryByDateComparator.INSTANCE);
entriesTwo.sort();
actualEntries.sort();
Assert.assertEquals(actualEntries, entriesTwo);
} else if (groupedBy.equals(Tags.create(key1, "three"))) {
Assert.assertEquals(groupResult.asList(), entriesThree);
Assert.assertEquals(groupResult.flatMap(), entriesThree);
} else {
Assert.fail("unexpected group: " + groupedBy);
}
@@ -256,11 +312,19 @@ public class PerformanceDbTest {
}
}
private List<Entry> storeEntries(final PerformanceDb performanceDb, final TimeRange timeRange,
private LongList storeEntries(final PerformanceDb performanceDb, final TimeRange timeRange,
final long numberOfEntries, final Tags tags, final int addToDate) {
final List<Entry> entries = generateEntries(timeRange, numberOfEntries, addToDate, tags);
performanceDb.put(entries);
return entries;
final LongList result = new LongList();
for (final Entry entry : entries) {
result.add(entry.getEpochMilli());
result.add(entry.getValue());
}
return result;
}
private void printEntries(final List<Entry> entriesOne, final String label) {
@@ -271,4 +335,15 @@ public class PerformanceDbTest {
index++;
}
}
private LongList toExpectedValues(final List<Entry> entries) {
final LongList result = new LongList();
for (final Entry entry : entries) {
result.add(entry.getEpochMilli());
result.add(entry.getValue());
}
return result;
}
}