new file format

Store values in sequences of variable length. Instead of using 8 bytes
per entry we are now using between 2 and 20 bytes. But we are also able
to store every non-negative long value.

pdb-api/src/main/java/org/lucares/pdb/api/Entry.java

@@ -13,8 +13,6 @@ public class Entry {
 	 */
 	public static final Entry POISON = new Entry(0, -1);
 
-	public static final long MAX_VALUE = 0xFF_FF_FF_FFL;
-
 	private final long epochMilli;
 
 	private final long value;
@@ -28,8 +26,8 @@ public class Entry {
 	}
 
 	public Entry(final long epochMilli, final long value, final Tags tags) {
-		if (value < 0 || value > MAX_VALUE) {
+		if (value < 0) {
-			throw new IllegalArgumentException("value must be between 0 and " + MAX_VALUE + ", but was " + value);
+			throw new IllegalArgumentException("value must be between 0 and " + Long.MAX_VALUE + ", but was " + value);
 		}
 
 		this.epochMilli = epochMilli;

pdb-api/src/main/java/org/lucares/pdb/api/Tags.java

@@ -10,7 +10,9 @@ import java.util.TreeSet;
 import java.util.function.BiConsumer;
 
 public class Tags {
-	static final Tags EMPTY = new Tags();
+	// TODO @ahr move class to org.lcuares.performance.db and make this package
+	// private
+	public static final Tags EMPTY = new Tags();
 
 	private final Map<String, Tag> tags;
 

performanceDb/.gitignore

@@ -4,3 +4,4 @@
 /.classpath
 /.project
 /test-output
+/doc/

performanceDb/src/main/java/org/lucares/performance/db/ByteType.java

@@ -0,0 +1,80 @@
+package org.lucares.performance.db;
+
+enum ByteType {
+
+	CONTINUATION(ContinuationByte.CONTINUATION_BYTE_PREFIX), // 10000000
+
+	MEASUREMENT(1 << 6), // 01000000
+
+	DATE_INCREMENT(1 << 5), // 00100000
+
+	DATE_OFFSET(1 << 4), // 00010000
+
+	VERSION(1);// 00000001
+
+	interface ContinuationByte {
+		long NUMBER_OF_VALUES_BITS = 7;
+
+		long CONTINUATION_BYTE_PREFIX = 1 << NUMBER_OF_VALUES_BITS; // 10000000
+	}
+
+	interface VersionByte {
+		/**
+		 * The version uses at least two bytes. The first byte is the prefix
+		 * which cannot hold any value (unless it is 0). And the second byte is
+		 * the actual value.
+		 */
+		long MIN_LENGTH = 2;
+	}
+
+	private final long firstBytePrefix;
+
+	private ByteType(final long firstBytePrefix) {
+		this.firstBytePrefix = firstBytePrefix;
+	}
+
+	public long getBytePrefix() {
+		return firstBytePrefix;
+	}
+
+	/**
+	 * the max value for the first byte is the prefix minus 1, because prefixes
+	 * start with 0⋯010⋯0, so prefix -1 is 0⋯01⋯1 which is exactly the max value
+	 *
+	 * @return the maximum value for the first byte of this sequence
+	 */
+	public long getFirstByteMaxValue() {
+		return firstBytePrefix - 1;
+	}
+
+	/**
+	 * the value bits are the prefix minus 1, because prefixes start with
+	 * 0⋯010⋯0, so prefix -1 is 0⋯01⋯1 which exactly represents the value bits.
+	 *
+	 * @return bitmap with the value bits set
+	 */
+	public long getValueBits() {
+		return firstBytePrefix - 1;
+	}
+
+	public boolean isValid(final int theByte) {
+
+		final long prefixBits = getPrefixBits();
+
+		return firstBytePrefix == (theByte & prefixBits);
+	}
+
+	private long getPrefixBits() {
+		return (~getValueBits()) & 0xff;
+	}
+
+	public static ByteType getType(final int aByte) {
+
+		for (final ByteType byteType : values()) {
+			if (byteType.isValid(aByte)) {
+				return byteType;
+			}
+		}
+		return null;
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/FileCorruptException.java

@@ -0,0 +1,10 @@
+package org.lucares.performance.db;
+
+public class FileCorruptException extends RuntimeException {
+
+	private static final long serialVersionUID = -4194021585305770924L;
+
+	public FileCorruptException(final String message) {
+		super(message);
+	}
+}

performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java

@@ -8,30 +8,17 @@ import org.lucares.pdb.api.Tags;
 class PdbFile {
 	private final Tags tags;
 
-	private final Day day;
-
 	private final Path path;
 
-	private final long offsetInEpochMilli;
-
 	@Deprecated
-	public PdbFile(final Day day, final File file, final Tags tags) {
+	public PdbFile(final File file, final Tags tags) {
-		this.day = day;
 		this.path = file.toPath();
 		this.tags = tags;
-		offsetInEpochMilli = day.getOffsetInEpochMilli();
 	}
 
-	public PdbFile(final Day day, final Path path, final Tags tags) {
+	public PdbFile(final Path path, final Tags tags) {
-		this.day = day;
 		this.path = path;
 		this.tags = tags;
-		offsetInEpochMilli = day.getOffsetInEpochMilli();
-	}
-
-	public static PdbFile today(final File file, final Tags tags) {
-		final Day day = new Day();
-		return new PdbFile(day, file, tags);
 	}
 
 	public Tags getTags() {
@@ -47,30 +34,15 @@ class PdbFile {
 		return path;
 	}
 
-	public Day getDay() {
-		return day;
-	}
-
-	public TimeRange getTimeRange() {
-
-		return day.toTimeRange();
-	}
-
-	public long getOffsetInEpochMilli() {
-		return offsetInEpochMilli;
-	}
-
 	@Override
 	public String toString() {
-		return "PdbFile [" + path + " " + getTimeRange() + " " + tags + "]\n";
+		return "PdbFile [" + path + " " + tags + "]\n";
 	}
 
 	@Override
 	public int hashCode() {
 		final int prime = 31;
 		int result = 1;
-		result = prime * result + ((day == null) ? 0 : day.hashCode());
-		result = prime * result + (int) (offsetInEpochMilli ^ (offsetInEpochMilli >>> 32));
 		result = prime * result + ((path == null) ? 0 : path.hashCode());
 		result = prime * result + ((tags == null) ? 0 : tags.hashCode());
 		return result;
@@ -85,13 +57,6 @@ class PdbFile {
 		if (getClass() != obj.getClass())
 			return false;
 		final PdbFile other = (PdbFile) obj;
-		if (day == null) {
-			if (other.day != null)
-				return false;
-		} else if (!day.equals(other.day))
-			return false;
-		if (offsetInEpochMilli != other.offsetInEpochMilli)
-			return false;
 		if (path == null) {
 			if (other.path != null)
 				return false;
@@ -104,4 +69,10 @@ class PdbFile {
 			return false;
 		return true;
 	}
+
+	public TimeRange getTimeRange() {
+		// TODO @ahr should return the minimal date that can be added
+		return null;
+	}
+
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java

@@ -3,15 +3,15 @@ package org.lucares.performance.db;
 import java.time.OffsetDateTime;
 import java.util.Comparator;
 
-public class PdbFileByTimeAsc implements Comparator<PdbFile> {
+public class PdbFileByTimeAsc implements Comparator<PdbFileOffsetTime> {
 
 	public static final PdbFileByTimeAsc INSTANCE = new PdbFileByTimeAsc();
 
 	@Override
-	public int compare(final PdbFile o1, final PdbFile o2) {
+	public int compare(final PdbFileOffsetTime o1, final PdbFileOffsetTime o2) {
 
-		final OffsetDateTime o1From = o1.getTimeRange().getFrom();
+		final OffsetDateTime o1From = o1.getOffsetTime();
-		final OffsetDateTime o2From = o2.getTimeRange().getFrom();
+		final OffsetDateTime o2From = o2.getOffsetTime();
 		return o1From.compareTo(o2From);
 	}
 

performanceDb/src/main/java/org/lucares/performance/db/PdbFileOffsetTime.java

@@ -0,0 +1,29 @@
+package org.lucares.performance.db;
+
+import java.time.OffsetDateTime;
+
+public class PdbFileOffsetTime {
+	private final PdbFile pdbFile;
+
+	private final OffsetDateTime offsetTime;
+
+	public PdbFileOffsetTime(final PdbFile pdbFile, final OffsetDateTime offsetTime) {
+		super();
+		this.pdbFile = pdbFile;
+		this.offsetTime = offsetTime;
+	}
+
+	public PdbFile getPdbFile() {
+		return pdbFile;
+	}
+
+	public OffsetDateTime getOffsetTime() {
+		return offsetTime;
+	}
+
+	@Override
+	public String toString() {
+		return "PdbFileOffsetTime [pdbFile=" + pdbFile + ", offsetTime=" + offsetTime + "]";
+	}
+
+}

performanceDb/src/main/java/org/lucares/performance/db/PdbFileUtils.java

@@ -5,17 +5,11 @@ import java.io.IOException;
 import java.time.OffsetDateTime;
 
 class PdbFileUtils {
-	static TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException {
+	static OffsetDateTime dateOffset(final PdbFile pdbFile) throws FileNotFoundException, IOException {
 
 		try (PdbReader reader = new PdbReader(pdbFile)) {
-			if (reader.canSeekTail(2)) {
+			reader.seekToLastValue();
-				reader.seekTail(2);
+			return reader.getDateOffsetAtCurrentPosition();
-				final OffsetDateTime lastWrittenDate = reader.readDate();
-
-				return new TimeRange(lastWrittenDate, pdbFile.getTimeRange().getTo());
-			} else {
-				return pdbFile.getTimeRange();
-			}
 		}
 	}
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java

@@ -1,12 +1,14 @@
 package org.lucares.performance.db;
 
+import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.RandomAccessFile;
+import java.io.InputStream;
 import java.time.Instant;
 import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
+import java.time.ZoneId;
 import java.util.Optional;
 
 import org.lucares.pdb.api.Entry;
@@ -14,134 +16,54 @@ import org.lucares.pdb.api.Tags;
 
 class PdbReader implements AutoCloseable {
 
-	private static final int BYTES_PER_VALUE = 4;
+	private static final int PEEK_NOT_SET = Integer.MIN_VALUE;
-	private final RandomAccessFile data;
+
-	private final byte[] buffer = new byte[BYTES_PER_VALUE];
+	static final long VERSION = 1;
-	private final PdbFile pdbFile;
+
+	private final InputStream data;
+	private long dateOffsetAtCurrentLocation = 0;
+	private long index = 0;
+	private int peekedByte = PEEK_NOT_SET;
 
 	public PdbReader(final PdbFile pdbFile) throws FileNotFoundException {
 		super();
-		this.pdbFile = pdbFile;
+		final File storageFile = pdbFile.getPath().toFile();
-		File storageFile = pdbFile.getPath().toFile();
+
-		this.data = new RandomAccessFile(storageFile, "r");
+		this.data = new BufferedInputStream(new FileInputStream(storageFile));
+
+		init();
 	}
 
-	/**
+	private void init() {
-	 * Reads the next value.
-	 * <p>
-	 * All values are non-negative. A negative return value indicates that the
-	 * end of the file has been reached
-	 *
-	 * @return the value or -1 if end of stream has been reached
-	 */
-	public long readValue() {
-		return read();
-	}
-
-	/**
-	 * Reads the next date value.
-	 *
-	 * @return the date, or {@code -1} if end of stream has been reached
-	 * @throws IOException
-	 */
-	public long readEpochMilli() {
-		final long value = read();
-		if (value < 0) {
-			return -1;
-		}
-		return pdbFile.getOffsetInEpochMilli() + value;
-	}
-
-	public OffsetDateTime readDate() {
-		final long epochMilli = readEpochMilli();
-
-		if (epochMilli < 0) {
-			return null;
-		}
-		return Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
-	}
-
-	// visible for test
-	long read() {
 		try {
-			final int read = data.read(buffer);
+			final long version = readValue(ByteType.VERSION);
-
+			if (version == -1) {
-			if (read != BYTES_PER_VALUE) {
+				throw new IllegalStateException("Cannot read empty file. The file must have at least a version. "
-				if (read < 0) {
+						+ "Otherwise we don't know in which version a writer might append data.");
-					return -1;
+			} else if (version != VERSION) {
-				} else {
+				throw new IllegalStateException(
-					throw new IllegalStateException("invalid file");
+						"The file is not of version " + VERSION + ". Actual version: " + version);
 			}
-			}
-			return BitFiddling.makeLong(buffer[0], buffer[1], buffer[2], buffer[3]);
 		} catch (final IOException e) {
-			throw new ReadRuntimeException(e);
+			throw new ReadException(e);
-		}
-	}
-
-	/**
-	 * Seek to the n-th value.
-	 *
-	 * @param n
-	 */
-	public void seek(final long n) {
-		try {
-			if (n < 0) {
-				throw new IllegalArgumentException("n must be non-negative, but was " + n);
-			}
-
-			final long pos = n * BYTES_PER_VALUE;
-
-			if (pos >= data.length()) {
-				throw new IllegalArgumentException("cannot seek to value " + n + ", because the file only has "
-						+ (data.length() / BYTES_PER_VALUE) + " values");
-			}
-
-			data.seek(pos);
-		} catch (final IOException e) {
-			throw new ReadRuntimeException(e);
 		}
 	}
 
 	/**
 	 * Seek to the end of the file.
-	 * <p>
-	 * After this operation you can read {@code n} values.
 	 *
-	 * @param n
-	 *            seek to the n-th last value
 	 * @throws ReadRuntimeException
-	 *             if the file does not have {@code n} entries
+	 *             if an IOException occurs
 	 */
-	public void seekTail(final long n) {
+	public void seekToLastValue() {
-		try {
-			if (n < 0) {
-				throw new IllegalArgumentException("n must be non-negative, but was " + n);
-			}
 
-			final long pos = computeSeekPosition(n);
+		while (readEntry(Tags.EMPTY).isPresent()) {
-
+			// seek to the end
-			data.seek(pos);
+			// TODO @ahr add date offsets every x kb, so we don't have
-		} catch (final IOException e) {
+			// to read the whole file
-			throw new ReadRuntimeException(e);
 		}
 	}
 
-	public boolean canSeekTail(final long n) {
-		try {
-			final long pos = computeSeekPosition(n);
-			return pos >= 0;
-		} catch (final IOException e) {
-			throw new ReadRuntimeException(e);
-		}
-	}
-
-	private long computeSeekPosition(final long n) throws IOException {
-		final long length = data.length();
-		final long pos = length - BYTES_PER_VALUE * n;
-		return pos;
-	}
-
 	@Override
 	public void close() {
 		try {
@@ -152,21 +74,92 @@ class PdbReader implements AutoCloseable {
 	}
 
 	Entry readNullableEntry(final Tags tags) throws ReadRuntimeException {
-		final long epochMilli = readEpochMilli();
+		try {
-		if (epochMilli < 0) {
+			final long epochMilliIncrement = readValue(ByteType.DATE_INCREMENT);
+			if (epochMilliIncrement < 0) {
 				return null;
 			}
-		final long value = readValue();
+			final long epochMilli = dateOffsetAtCurrentLocation + epochMilliIncrement;
+			final long value = readValue(ByteType.MEASUREMENT);
 
 			if (value < 0) {
 				return null;
 			}
+			dateOffsetAtCurrentLocation = epochMilli;
 			return new Entry(epochMilli, value, tags);
+		} catch (final IOException e) {
+			throw new ReadException(e);
+		}
 	}
 
 	public Optional<Entry> readEntry(final Tags tags) throws ReadRuntimeException {
 
-		return Optional.ofNullable(readNullableEntry(tags));
+		final Entry entry = readNullableEntry(tags);
+		return Optional.ofNullable(entry);
+	}
+
+	public OffsetDateTime getDateOffsetAtCurrentPosition() {
+		return OffsetDateTime.ofInstant(Instant.ofEpochMilli(dateOffsetAtCurrentLocation), ZoneId.of("UTC"));
+	}
+
+	public long readValue(final ByteType byteType) throws IOException {
+
+		final long firstByteValueBits = byteType.getValueBits();
+
+		int firstByte = readNextByte();
+
+		if (!byteType.isValid(firstByte)) {
+			if (firstByte < 0) {
+				return -1;
+			} else if (ByteType.DATE_OFFSET.isValid(firstByte)) {
+				final long dateOffsetInit = firstByte & ByteType.DATE_OFFSET.getValueBits();
+				this.dateOffsetAtCurrentLocation = readContinuationBytes(dateOffsetInit);
+				firstByte = readNextByte();
+			} else {
+				throw new FileCorruptException(
+						"File corrupt at " + index + ". Byte type was " + ByteType.getType(firstByte));
+			}
+		}
+
+		final long value = firstByte & firstByteValueBits;
+
+		return readContinuationBytes(value);
+	}
+
+	private int readNextByte() throws IOException {
+
+		final int result;
+		if (peekedByte == PEEK_NOT_SET) {
+			result = data.read();
+		} else {
+			result = peekedByte;
+			peekedByte = PEEK_NOT_SET;
+		}
+		index++;
+		return result;
+	}
+
+	private int peekNextByte() throws IOException {
+		if (peekedByte == PEEK_NOT_SET) {
+			peekedByte = data.read();
+		}
+		return peekedByte;
+	}
+
+	private long readContinuationBytes(long value) throws IOException {
+		int nextByte;
+		while ((nextByte = peekNextByte()) >= 0 && isContinuationByte(nextByte)) {
+			value = value << ByteType.ContinuationByte.NUMBER_OF_VALUES_BITS;
+			value = value | (nextByte & ByteType.CONTINUATION.getValueBits());
+			readNextByte();
+		}
+
+		return value;
+	}
+
+	private static boolean isContinuationByte(final int nextByte) {
+		final long bytePrefix = ByteType.CONTINUATION.getBytePrefix();
+		return bytePrefix == (nextByte & bytePrefix);
 	}
 
 }

performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java

@@ -5,18 +5,82 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.time.OffsetDateTime;
 import java.util.logging.Logger;
 
 import org.lucares.pdb.api.Entry;
 
+/**
+ * File format description:
+ * <p>
+ * We store non-negative long values for epoch milli and a measurement (usually
+ * duration in ms). Both values are stored as pairs, so that we get
+ * date-measurement-date-measurement-date... . The date values are stored as
+ * difference to the previous date. Every few kilobytes we add an absolute
+ * offset, so that we can synchronize and don't have to read the whole file when
+ * we want to append.
+ *
+ * <p>
+ * For example we want to store the following values:
+ *
+ * <pre>
+ * 2009-02-45T12:31:30.30+0100 123 the date is 1234567890 in epoch millis
+ * 2009-02-45T01:06:39.39+0100 456 the date is 1234569999 in epoch millis
+ * </pre>
+ * <p>
+ * We would first store the offset 1234567890, then die first pair. The date is
+ * stored as the offset to the last value (which was the offset), so it is 0.
+ * Then we store the measurement. Next we store the second pair. The date
+ * difference is 2109 and the measurement is 456.
+ * <p>
+ * Each value is stored with a variable length byte sequence. The idea is
+ * similar to the encoding of UTF-8. But we differentiate between several
+ * different types of values.
+ * <ol>
+ * <li>version, start with 000001
+ * <li>number of entries up until this point in this file, 00001
+ * <li>date offsets with absolute values for epoch milli, start with 0001
+ * <li>date increments to the previous date value, start with 001
+ * <li>measurements, start with 01
+ * <li>continuation bytes, start with 1
+ * </ol>
+ *
+ * This is different from UTF-8. We do not encode the number of continuation
+ * bytes. Therefore we loose UTF-8's self validation feature and we cannot skip
+ * to the next value without reading all continuation bytes. But it is a little
+ * bit more efficient, because each continuation byte can store 7 bit instead of
+ * 6. A four byte sequence in UTF-8 can store 21 bits whereas a four byte
+ * sequence in this scheme stores 27 bits for values and 26 bits for date
+ * increments. But it is not as efficent for one byte sequences. On the other
+ * hand we also encode five different value types.
+ * <p>
+ * The encoding looks as follows:
+ * <p>
+ * The first byte starts with 00001 for meta-data. The three remaining bits are
+ * used for the version number. 001 in our case. So the first byte looks like
+ * this. 00001001
+ * <p>
+ * The second byte starts with 0001 for date offsets, 001 for date increments
+ * and 01 for measurements. All continuation bytes start with 1. E.g. The
+ * measurement 202 has the unsigned bit representation 11001010. The first byte
+ * of a measurement value starts with 01, so we have room for the first 6 bits.
+ * But we need 8 bits. So we must add another byte. The second byte starts with
+ * 1 and has room for 7 bits. The result looks like this: <b>01</b><i>000001<i>
+ * <b>1</b><i>1001010</i>
+ */
 class PdbWriter implements AutoCloseable {
 
 	private final static Logger LOGGER = Logger.getLogger(PdbWriter.class.getCanonicalName());
 
 	private static final boolean APPEND = true;
+
+	private static final int MAX_BYTES_PER_VALUE = 10;
+
+	private final byte[] buffer = new byte[MAX_BYTES_PER_VALUE];
+
 	private final OutputStream outputStream;
 	private final PdbFile pdbFile;
-	private long minimalEpochMilli;
+	private long lastEpochMilli;
 
 	PdbWriter(final PdbFile pdbFile) throws IOException {
 		this.pdbFile = pdbFile;
@@ -24,10 +88,15 @@ class PdbWriter implements AutoCloseable {
 		this.outputStream = new BufferedOutputStream(new FileOutputStream(storageFile, APPEND));
 
 		if (storageFile.exists() && storageFile.length() > 0) {
-			final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
+			// TODO @ahr check version
-			minimalEpochMilli = availableTimeRange.getFrom().toInstant().toEpochMilli();
+
+			final OffsetDateTime dateOffset = PdbFileUtils.dateOffset(pdbFile);
+			lastEpochMilli = dateOffset.toInstant().toEpochMilli();
 		} else {
-			minimalEpochMilli = pdbFile.getTimeRange().getFrom().toInstant().toEpochMilli();
+			writeValue(PdbReader.VERSION, ByteType.VERSION, outputStream);
+			writeValue(0, ByteType.DATE_OFFSET, outputStream);
+
+			lastEpochMilli = 0;
 		}
 	}
 
@@ -36,24 +105,38 @@ class PdbWriter implements AutoCloseable {
 	}
 
 	public void write(final Entry entry) throws WriteException {
-		write(entry.getEpochMilli(), entry.getValue());
+		final long epochMilli = entry.getEpochMilli();
+		final long value = entry.getValue();
+		write(epochMilli, value);
 	}
 
 	private void write(final long epochMilli, final long value) throws WriteException {
-		final long offsetEpochMilli = pdbFile.getOffsetInEpochMilli();
+		try {
-		final long adjustedValue = epochMilli - offsetEpochMilli;
+
-		assertValueInRange(adjustedValue);
+			if (epochMilli < lastEpochMilli) {
+				LOGGER.info("epochMilli must not be smaller than " + lastEpochMilli + ", but was " + epochMilli
+						+ ". We'll accept this for now. "
+						+ "Currently there is no code that relies on monotonically increasing date values. "
+						+ "Log4j does not guarantee it either.");
+				return;
+			}
+
+			final long epochMilliIncrement = epochMilli - lastEpochMilli;
+			assertValueInRange(epochMilliIncrement);
 			assertValueInRange(value);
 			assertEpochMilliInRange(epochMilli);
 
-		write(adjustedValue);
+			writeValue(epochMilliIncrement, ByteType.DATE_INCREMENT, outputStream);
-		write(value);
+			writeValue(value, ByteType.MEASUREMENT, outputStream);
-		minimalEpochMilli = epochMilli;
+			lastEpochMilli = epochMilli;
+		} catch (final IOException e) {
+			throw new WriteException(e);
+		}
 	}
 
 	private void assertEpochMilliInRange(final long epochMilli) {
-		if (epochMilli < minimalEpochMilli) {
+		if (epochMilli < lastEpochMilli) {
-			LOGGER.fine("epochMilli must not be smaller than " + minimalEpochMilli + ", but was " + epochMilli
+			LOGGER.info("epochMilli must not be smaller than " + lastEpochMilli + ", but was " + epochMilli
 					+ ". We'll accept this for now. "
 					+ "Currently there is no code that relies on monotonically increasing date values. "
 					+ "Log4j does not guarantee it either.");
@@ -64,30 +147,6 @@ class PdbWriter implements AutoCloseable {
 		if (value < 0) {
 			throw new IllegalArgumentException("value must not be negative: " + value);
 		}
-		if (value > Entry.MAX_VALUE) {
-			throw new IllegalArgumentException("max value is " + Entry.MAX_VALUE + " value was: " + value);
-		}
-	}
-
-	// visible for test
-	void write(final long value) throws WriteException {
-		assertValueInRange(value);
-		try {
-			outputStream.write(BitFiddling.long3(value));
-			outputStream.write(BitFiddling.long2(value));
-			outputStream.write(BitFiddling.long1(value));
-			outputStream.write(BitFiddling.long0(value));
-		} catch (final IOException e) {
-			throw new WriteException(e);
-		}
-	}
-
-	public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException {
-		try (PdbWriter writer = new PdbWriter(pdbFile)) {
-			for (final Entry entry : entries) {
-				writer.write(entry);
-			}
-		}
 	}
 
 	@Override
@@ -99,4 +158,36 @@ class PdbWriter implements AutoCloseable {
 	public void flush() throws IOException {
 		outputStream.flush();
 	}
+
+	public void writeValue(final long value, final ByteType byteSequenceType, final OutputStream output)
+			throws IOException {
+
+		int index = buffer.length - 1;
+
+		final long maxFirstByteValue = byteSequenceType.getFirstByteMaxValue();
+		long val = value;
+		while (val > maxFirstByteValue) {
+			// handles continuation bytes
+			buffer[index] = (byte) ((val & ByteType.CONTINUATION.getValueBits())
+					| ByteType.CONTINUATION.getBytePrefix());
+			index--;
+			val = val >> ByteType.ContinuationByte.NUMBER_OF_VALUES_BITS;
+		}
+
+		buffer[index] = (byte) (val | byteSequenceType.getBytePrefix());
+
+		output.write(buffer, index, buffer.length - index);
+	}
+
+	public static void writeEntry(final PdbFile pdbFile, final Entry... entries) throws IOException {
+		try (PdbWriter writer = new PdbWriter(pdbFile)) {
+			for (final Entry entry : entries) {
+				writer.write(entry);
+			}
+		}
+	}
+
+	public static void init(final PdbFile result) throws IOException {
+		writeEntry(result);
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java

@@ -7,32 +7,34 @@ import org.lucares.pdb.api.Tags;
 
 public class StorageUtils {
 
-	public static Path createStorageFile(final Path tagSpecificStorageFolder, final Day day) {
+	public static Path createStorageFile(final Path tagSpecificStorageFolder) {
 
-		final Path dateSpecificFolder = tagSpecificStorageFolder.resolve(day.format("/"));
+		final Path storageFile = tagSpecificStorageFolder.resolve(UUID.randomUUID().toString());
-		final Path storageFile = dateSpecificFolder.resolve(UUID.randomUUID().toString());
 
 		return storageFile;
 	}
 
-	public static Day getDateOffset(final Path pathToStorageFile) {
+	// TODO @ahr remove
-
+	// public static Day getDateOffset(final Path pathToStorageFile) {
-		try {
+	//
-			final Path pathDay = pathToStorageFile.getParent();
+	// try {
-			final Path pathMonth = pathDay.getParent();
+	// final Path pathDay = pathToStorageFile.getParent();
-			final Path pathYear = pathMonth.getParent();
+	// final Path pathMonth = pathDay.getParent();
-
+	// final Path pathYear = pathMonth.getParent();
-			final int day = Integer.parseInt(pathDay.getFileName().toString(), 10);
+	//
-			final int month = Integer.parseInt(pathMonth.getFileName().toString(), 10);
+	// final int day = Integer.parseInt(pathDay.getFileName().toString(), 10);
-			final int year = Integer.parseInt(pathYear.getFileName().toString(), 10);
+	// final int month = Integer.parseInt(pathMonth.getFileName().toString(),
-
+	// 10);
-			final Day result = new Day(year, month, day);
+	// final int year = Integer.parseInt(pathYear.getFileName().toString(), 10);
-			return result;
+	//
-		} catch (final NumberFormatException e) {
+	// final Day result = new Day(year, month, day);
-			throw new IllegalStateException(pathToStorageFile.toUri().getPath() + " is not a path to a storage file",
+	// return result;
-					e);
+	// } catch (final NumberFormatException e) {
-		}
+	// throw new IllegalStateException(pathToStorageFile.toUri().getPath() + "
-	}
+	// is not a path to a storage file",
+	// e);
+	// }
+	// }
 
 	public static Path createTagSpecificStorageFolder(final Path dataDirectory, final Tags tags) {
 
@@ -47,9 +49,6 @@ public class StorageUtils {
 	public static Path getTagSpecificStorageFolder(final Path storageFilePath) {
 
 		return storageFilePath //
-				.getParent() // day
-				.getParent() // month
-				.getParent() // year
 				.getParent(); // tag specific
 	}
 }

performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java

@@ -6,6 +6,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -67,8 +68,7 @@ public class TagsToFile implements CollectionUtils {
 			final List<Path> storageFiles = FileUtils.listRecursively(tagSpecific.getPath());
 			for (final Path storageFile : storageFiles) {
 
-				final Day day = StorageUtils.getDateOffset(storageFile);
+				final PdbFile pdbFile = new PdbFile(storageFile, fileSpecificTags);
-				final PdbFile pdbFile = new PdbFile(day, storageFile, fileSpecificTags);
 
 				result.add(pdbFile);
 			}
@@ -110,21 +110,20 @@ public class TagsToFile implements CollectionUtils {
 	public PdbFile getFile(final OffsetDateTime date, final Tags tags) throws FileNotFoundException, IOException {
 
 		final List<PdbFile> pdbFiles = getFilesMatchingTagsExactly(tags);
-		final List<PdbFile> preResult = new ArrayList<>();
+		final List<PdbFileOffsetTime> preResult = new ArrayList<>();
 
 		assertAllFilesHaveSameFolder(pdbFiles);
 
 		PdbFile result;
 		for (final PdbFile pdbFile : pdbFiles) {
 
-			// TODO @ahr should compare with the last written date
+			if (Files.isRegularFile(pdbFile.getPath())
-			final boolean inRange = pdbFile.getTimeRange().inRange(date);
+					&& Files.size(pdbFile.getPath()) >= ByteType.VersionByte.MIN_LENGTH) {
 
-			if (inRange) {
+				final OffsetDateTime offsetTime = PdbFileUtils.dateOffset(pdbFile);
-				final TimeRange availableTimeRange = PdbFileUtils.getAvailableTimeRange(pdbFile);
 
-				if (availableTimeRange.inRange(date)) {
+				if (!offsetTime.isAfter(date)) {
-					preResult.add(pdbFile);
+					preResult.add(new PdbFileOffsetTime(pdbFile, offsetTime));
 				}
 			}
 		}
@@ -139,9 +138,10 @@ public class TagsToFile implements CollectionUtils {
 				tagSpecificStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFilePath);
 			}
 
-			result = createNewPdbFile(date, tags, tagSpecificStorageFolder);
+			result = createNewPdbFile(tags, tagSpecificStorageFolder);
 		} else {
-			result = preResult.get(0);
+			Collections.sort(preResult, PdbFileByTimeAsc.INSTANCE.reversed());
+			result = preResult.get(0).getPdbFile();
 		}
 
 		return result;
@@ -159,11 +159,10 @@ public class TagsToFile implements CollectionUtils {
 		}
 	}
 
-	private PdbFile createNewPdbFile(final OffsetDateTime date, final Tags tags, final Path tagSpecificStorageFolder) {
+	private PdbFile createNewPdbFile(final Tags tags, final Path tagSpecificStorageFolder) throws IOException {
 		final Path storageFile;
 		PdbFile result;
-		storageFile = createNewFile(date, tagSpecificStorageFolder);
+		storageFile = createNewFile(tagSpecificStorageFolder);
-		final Day day = new Day(date);
 
 		final Document document = db.getDocument(tagSpecificStorageFolder.toFile());
 		if (document == null) {
@@ -174,14 +173,14 @@ public class TagsToFile implements CollectionUtils {
 			});
 		}
 
-		result = new PdbFile(day, storageFile, tags);
+		result = new PdbFile(storageFile, tags);
+		PdbWriter.init(result);
 		return result;
 	}
 
-	private Path createNewFile(final OffsetDateTime date, final Path tagSpecificStorageFolder) {
+	private Path createNewFile(final Path tagSpecificStorageFolder) {
-		final Day day = new Day(date);
 
-		final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
+		final Path result = StorageUtils.createStorageFile(tagSpecificStorageFolder);
 		try {
 			Files.createDirectories(result.getParent());
 			Files.createFile(result);

performanceDb/src/test/java/org/lucares/performance/db/BitFiddlingTest.java

@@ -0,0 +1,42 @@
+package org.lucares.performance.db;
+
+import org.testng.annotations.Test;
+
+@Test
+public class BitFiddlingTest {
+
+	// TODO @ahr remove or move
+	// public void testEncodingMeasurement() throws Exception {
+	//
+	// final List<ByteType> types = Arrays.asList(ByteType.DATE_INCREMENT,
+	// ByteType.DATE_OFFSET, ByteType.MEASUREMENT,
+	// ByteType.VERSION);
+	//
+	// final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L,
+	// 202L, 255L, 256L, 8191L, 8192L, 1048575L,
+	// 1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L,
+	// 2199023255552L, 281474976710655L,
+	// 281474976710656L, 36028797018963967L, 36028797018963968L,
+	// 4611686018427387901L, 4611686018427387904L);
+	//
+	// for (final Long value : values) {
+	// for (final ByteType type : types) {
+	// encodeDecode(value, type);
+	// }
+	// }
+	// }
+	//
+	// private void encodeDecode(final Long value, final ByteType byteType)
+	// throws Exception {
+	//
+	// final ByteArrayOutputStream output = new ByteArrayOutputStream();
+	// BitFiddling.writeValue(value, byteType, output);
+	//
+	// final byte[] byteArray = output.toByteArray();
+	// final ByteArrayInputStream input = new ByteArrayInputStream(byteArray);
+	// final Long readValue = BitFiddling.readValue(byteType, input);
+	//
+	// Assert.assertEquals(readValue, value);
+	// }
+
+}

performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java

@@ -4,7 +4,13 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
 
 import org.lucares.pdb.api.Entry;
 import org.lucares.pdb.api.Tags;
@@ -19,6 +25,8 @@ public class PdbReaderWriterTest {
 
 	private Path dataDirectory;
 
+	private static final Tags TAGS = Tags.create();
+
 	@BeforeMethod
 	public void beforeMethod() throws IOException {
 		dataDirectory = Files.createTempDirectory("pdb");
@@ -30,90 +38,57 @@ public class PdbReaderWriterTest {
 	}
 
 	@DataProvider(name = "providerWriteRead")
-	public Object[][] providerWriteRead() {
+	public Iterator<Object[]> providerWriteRead() {
-		return new Object[][] { //
+
-				{ 1 }, //
+		final OffsetDateTime two_sixteen = DateUtils.getDate(2016, 1, 1, 1, 1, 1);
-				{ 6 }, //
+
-				{ 0xffffffffL },//
+		final List<Long> values = Arrays.asList(0L, 1L, 63L, 64L, 127L, 128L, 202L, 255L, 256L, 8191L, 8192L, 1048575L,
-		};
+				1048576L, 134217728L, 17179869183L, 17179869184L, 2199023255551L, 2199023255552L, 281474976710655L,
+				281474976710656L, 36028797018963967L, 36028797018963968L, 4611686018427387901L, 4611686018427387904L);
+
+		final List<Object[]> result = new ArrayList<>();
+
+		// single values
+		for (final Long value : values) {
+			result.add(new Object[] { Arrays.asList(new Entry(two_sixteen, value, TAGS)) });
+		}
+
+		// multivalues
+		result.clear(); // TODO @ahr remove this line
+		final List<Entry> entries = new ArrayList<>();
+		for (int i = 0; i < 100; i++) {
+
+			final long epochMilli = 123456 * i;
+
+			final OffsetDateTime date = OffsetDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC"));
+
+			entries.add(new Entry(date, i, TAGS));
+		}
+		result.add(new Object[] { entries });
+
+		return result.iterator();
 	}
 
 	@Test(dataProvider = "providerWriteRead")
-	public void testWriteRead(final long value) throws Exception {
+	public void testWriteRead(final List<Entry> entries) throws Exception {
 
 		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final Tags tags = Tags.create();
+		final PdbFile pdbFile = new PdbFile(file.toPath(), TAGS);
-		final PdbFile pdbFile = PdbFile.today(file, tags);
-		final OffsetDateTime now = OffsetDateTime.now(); // TODO @ahr might fail
-															// at midnight
-		final Entry entry = new Entry(now, value, tags);
 
 		try (PdbWriter writer = new PdbWriter(pdbFile)) {
+			for (final Entry entry : entries) {
 				writer.write(entry);
 			}
+		}
 
 		try (final PdbReader reader = new PdbReader(pdbFile)) {
-			final Entry actual = reader.readEntry(tags).orElseThrow(() -> new AssertionError());
+
+			for (final Entry entry : entries) {
+
+				final Entry actual = reader.readEntry(TAGS).orElseThrow(() -> new AssertionError());
 
 				Assert.assertEquals(actual, entry);
 			}
 		}
-
-	public void testSeekTail() throws Exception {
-
-		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final PdbFile pdbFile = PdbFile.today(file, Tags.create());
-
-		try (PdbWriter writer = new PdbWriter(pdbFile)) {
-			writer.write(1);
-			writer.write(2);
-			writer.write(3);
-			writer.write(4);
-			writer.write(5);
-		}
-
-		try (final PdbReader reader = new PdbReader(pdbFile)) {
-			reader.seekTail(2);
-
-			final long four = reader.read();
-			final long five = reader.read();
-
-			Assert.assertEquals(four, 4, "second last value");
-			Assert.assertEquals(five, 5, "last value");
-
-			final long eof = reader.read();
-			Assert.assertEquals(eof, -1, "end of file");
-		}
-	}
-
-	public void testSeek() throws Exception {
-
-		final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile();
-		final PdbFile pdbFile = PdbFile.today(file, Tags.create());
-
-		try (PdbWriter writer = new PdbWriter(pdbFile)) {
-			writer.write(1);
-			writer.write(2);
-			writer.write(3);
-			writer.write(4);
-			writer.write(5);
-		}
-
-		try (final PdbReader reader = new PdbReader(pdbFile)) {
-			reader.seek(2);
-
-			final long three = reader.read();
-			final long four = reader.read();
-			final long five = reader.read();
-
-			Assert.assertEquals(three, 3, "third value");
-			Assert.assertEquals(four, 4, "fourth value");
-			Assert.assertEquals(five, 5, "fifth value");
-
-			reader.seek(0);
-			final long first = reader.read();
-			Assert.assertEquals(first, 1, "first value");
-
-		}
 	}
 }

performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java

@@ -34,7 +34,7 @@ public class PdbWriterManagerTest {
 			Path path;
 			try {
 				path = Files.createTempFile(dataDirectory, "pdb", ".data");
-				return new PdbWriter(new PdbFile(new Day(date), path, tags));
+				return new PdbWriter(new PdbFile(path, tags));
 			} catch (final IOException e) {
 				throw new AssertionError(e.getMessage(), e);
 			}

performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java

@@ -115,10 +115,9 @@ public class PerformanceDbTest {
 					.collect(Collectors.toList());
 			Assert.assertEquals(foldersInStorage.size(), 1);
 
-			final Path dateSpecificFolder = foldersInStorage.get(0)
+			final Path tagSpecificFolder = foldersInStorage.get(0);
-					.resolve(new Day(timeRange.getFrom()).format(File.separator));
 
-			final File[] filesInStorage = dateSpecificFolder.toFile().listFiles();
+			final File[] filesInStorage = tagSpecificFolder.toFile().listFiles();
 			Assert.assertEquals(filesInStorage.length, 1,
 					"one file in storage, but was: " + Arrays.asList(filesInStorage));
 		}

performanceDb/src/test/java/org/lucares/performance/db/StorageUtilsTest.java

@@ -10,30 +10,16 @@ import org.testng.annotations.Test;
 @Test
 public class StorageUtilsTest {
 
-	public void testExtractDateOffsetFromPath() throws Exception {
-
-		final Path tagSpecificStorageFolder = Paths.get("/tmp");
-		final Day day = new Day(2016, 1, 1);
-
-		final Path storageFile = StorageUtils.createStorageFile(tagSpecificStorageFolder, day);
-
-		final Day extractedDateOffset = StorageUtils.getDateOffset(storageFile);
-
-		Assert.assertEquals(extractedDateOffset, day);
-	}
-
 	public void testGetTagSpecificStorageFolder() {
 		final Path dataDirectory = Paths.get("/tmp");
 		final Tags tags = Tags.create("key", "value");
-		final Day day = new Day(2016, 1, 1);
 
 		final Path tagSpecifiStorageFolder = StorageUtils.createTagSpecificStorageFolder(dataDirectory, tags);
 
-		final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder, day);
+		final Path storageFile = StorageUtils.createStorageFile(tagSpecifiStorageFolder);
 
 		final Path extractedTagSpecifiStorageFolder = StorageUtils.getTagSpecificStorageFolder(storageFile);
 
 		Assert.assertEquals(extractedTagSpecifiStorageFolder, extractedTagSpecifiStorageFolder);
 	}
-
 }

performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java

@@ -40,13 +40,15 @@ public class TagsToFilesTest {
 			final Tags tags = Tags.create("myKey", "myValue");
 
 			final PdbFile newFileForTags = tagsToFile.getFile(date, tags);
+			PdbWriter.writeEntry(newFileForTags);
+
 			final PdbFile existingFileForTags = tagsToFile.getFile(date, tags);
 
 			Assert.assertEquals(newFileForTags, existingFileForTags);
 		}
 	}
 
-	public void testMultipleFilesForTag() throws Exception {
+	public void testAppendingToSameFileIfNewDateIsAfter() throws Exception {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
 
@@ -60,13 +62,14 @@ public class TagsToFilesTest {
 			final PdbFile fileForDay1 = tagsToFile.getFile(day1, tags);
 			final PdbFile fileForDay2 = tagsToFile.getFile(day2, tags);
 
-			Assert.assertNotEquals(fileForDay1, fileForDay2);
+			Assert.assertEquals(fileForDay1, fileForDay2);
 
 			final PdbFile existingFileForDay1 = tagsToFile.getFile(day1, tags);
 			Assert.assertEquals(fileForDay1, existingFileForDay1);
 		}
 	}
 
+	@Test(invocationCount = 1)
 	public void testNewFileIfDateIsTooOld() throws Exception {
 
 		try (H2DB db = new H2DB(new File(dataDirectory.toFile(), "lu.db"))) {
@@ -84,12 +87,12 @@ public class TagsToFilesTest {
 			PdbWriter.writeEntry(fileAfternoon, new Entry(afternoon, 1, tags));
 
 			final PdbFile fileMorning = tagsToFile.getFile(morning, tags);
-			PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
+			PdbWriter.writeEntry(fileMorning, new Entry(morning, 2, tags));
 
 			Assert.assertNotEquals(fileAfternoon, fileMorning);
 
 			final PdbFile fileEarlyMorning = tagsToFile.getFile(earlyMorning, tags);
-			PdbWriter.writeEntry(fileMorning, new Entry(morning, 1, tags));
+			PdbWriter.writeEntry(fileEarlyMorning, new Entry(earlyMorning, 3, tags));
 
 			Assert.assertNotEquals(fileEarlyMorning, fileAfternoon);
 			Assert.assertNotEquals(fileEarlyMorning, fileMorning);