tags are now stored as variable length byte sequences of longs
Replaced Tags.filenameBytes with a SortedSet<Tag>. Tags are now stored as longs (variable length encoded) in the PersistenMap. Tags.filenameBytes was introduced to reduce memory consumption, when all tags were hold in memory. Tags are now stored in a PersistentMap and only read when needed. Moved the VariableByteEncoder into its own project, because it was needed by pdb-api.
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
package org.lucares.utils.byteencoder;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.lucares.collections.LongList;
|
||||
import org.lucares.utils.byteencoder.VariableByteEncoder;
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.DataProvider;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
@Test
|
||||
public class VariableByteEncoderTest {
|
||||
|
||||
@DataProvider
|
||||
public Object[][] providerEncodeDecode() {
|
||||
return new Object[][] { //
|
||||
// encoded into 1 byte
|
||||
{ 10, -5, 5 }, //
|
||||
{ 10, 0, 5 }, //
|
||||
{ 10, -63, 63 }, //
|
||||
// encoded into 2 bytes
|
||||
{ 10, 130, 131 }, //
|
||||
// encoded into 3 bytes
|
||||
{ 10, -8191, 8191 }, //
|
||||
// encoded into n bytes
|
||||
{ 1, Long.MAX_VALUE / 2 - 4, Long.MAX_VALUE / 2 }, //
|
||||
{ 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2 }, //
|
||||
{ 11, Long.MIN_VALUE / 2 + 1, Long.MIN_VALUE / 2 + 3 }, //
|
||||
{ 12, Long.MAX_VALUE / 2 - 3, Long.MAX_VALUE / 2 },//
|
||||
};
|
||||
}
|
||||
|
||||
@Test(dataProvider = "providerEncodeDecode")
|
||||
public void testEncodeDecode(final long numValues, final long minValue, final long maxValue) {
|
||||
|
||||
final LongList originalValues = new LongList();
|
||||
final byte[] buffer = new byte[1024];
|
||||
final AtomicInteger offsetInBuffer = new AtomicInteger(0);
|
||||
|
||||
ThreadLocalRandom.current().longs(numValues, minValue, maxValue).forEachOrdered(value -> {
|
||||
originalValues.add(value);
|
||||
final int appendedBytes = VariableByteEncoder.encodeInto(value, buffer, offsetInBuffer.get());
|
||||
offsetInBuffer.addAndGet(appendedBytes);
|
||||
});
|
||||
|
||||
final LongList actualValues = VariableByteEncoder.decode(buffer);
|
||||
|
||||
assertEquals(actualValues.toString(), originalValues.toString());
|
||||
}
|
||||
|
||||
@DataProvider
|
||||
public Object[][] providerEncodeDecodeOfTwoValues() {
|
||||
return new Object[][] { //
|
||||
{ 12345, 67890, false, 1 }, // first value needs three bytes, it does not fit
|
||||
{ 12345, 67890, false, 2 }, // first value needs three bytes, it does not fit
|
||||
{ 12345, 67890, false, 3 }, // first value needs three bytes, second value does not fit
|
||||
{ 12345, 67890, false, 4 }, // first value needs three bytes, second value does not fit
|
||||
{ 12345, 67890, false, 5 }, // first value needs three bytes, second value does not fit
|
||||
{ 12345, 67890, true, 6 }, // both values need three bytes
|
||||
{ 12345, 67890, true, 10 }, //
|
||||
};
|
||||
}
|
||||
|
||||
@Test(dataProvider = "providerEncodeDecodeOfTwoValues")
|
||||
public void testEncodeDecodeOfTwoValues(final long value1, final long value2, final boolean fits,
|
||||
final int bufferSize) {
|
||||
final LongList originalValues = new LongList();
|
||||
final byte[] buffer = new byte[bufferSize];
|
||||
|
||||
final int bytesAdded = VariableByteEncoder.encodeInto(value1, value2, buffer, 0);
|
||||
Assert.assertEquals(bytesAdded > 0, fits);
|
||||
if (fits) {
|
||||
originalValues.addAll(value1, value2);
|
||||
} else {
|
||||
Assert.assertEquals(buffer, new byte[bufferSize],
|
||||
"checks that buffer is resetted after it discovers the values do not fit");
|
||||
}
|
||||
|
||||
final LongList decodedValues = VariableByteEncoder.decode(buffer);
|
||||
Assert.assertEquals(decodedValues, originalValues);
|
||||
}
|
||||
|
||||
@DataProvider
|
||||
public Object[][] providerNededBytes() {
|
||||
return new Object[][] { //
|
||||
{ 0, 1 }, //
|
||||
{ -10, 1 }, //
|
||||
{ 10, 1 }, //
|
||||
{ -63, 1 }, //
|
||||
{ 63, 1 }, //
|
||||
{ -64, 2 }, //
|
||||
{ 64, 2 }, //
|
||||
{ -8191, 2 }, //
|
||||
{ 8191, 2 }, //
|
||||
{ -8192, 3 }, //
|
||||
{ 8192, 3 }, //
|
||||
};
|
||||
}
|
||||
|
||||
@Test(dataProvider = "providerNededBytes")
|
||||
public void testNeededBytes(final long value, final int expectedNeededBytes) {
|
||||
|
||||
final int neededBytes = VariableByteEncoder.neededBytes(value);
|
||||
final byte[] encoded = VariableByteEncoder.encode(value);
|
||||
Assert.assertEquals(encoded.length, neededBytes);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user