tags are now stored as variable length byte sequences of longs

Replaced Tags.filenameBytes with a SortedSet<Tag>. Tags are now
stored as longs (variable length encoded) in the PersistenMap.
Tags.filenameBytes was introduced to reduce memory consumption, when
all tags were hold in memory. Tags are now stored in a PersistentMap
and only read when needed.

Moved the VariableByteEncoder into its own project, because it was
needed by pdb-api.
This commit is contained in:
2018-11-17 20:03:46 +01:00
parent b2107acf4e
commit 135ab42cd8
14 changed files with 97 additions and 128 deletions

View File

@@ -19,7 +19,6 @@ import org.lucares.pdb.api.StringCompressor;
import org.lucares.pdb.api.Tag;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.blockstorage.intsequence.VariableByteEncoder;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.lang.Expression;
@@ -29,6 +28,7 @@ import org.lucares.pdb.diskstorage.DiskStorage;
import org.lucares.pdb.map.PersistentMap;
import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
import org.lucares.utils.Preconditions;
import org.lucares.utils.byteencoder.VariableByteEncoder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,13 +52,13 @@ public class DataStore implements AutoCloseable {
private static final EncoderDecoder<Tags> ENCODER_TAGS = new EncoderDecoder<>() {
@Override
public byte[] encode(final Tags object) {
return object.getFilenameBytes();
public byte[] encode(final Tags tags) {
return tags.toBytes();
}
@Override
public Tags decode(final byte[] bytes) {
return new Tags(bytes);
return Tags.fromBytes(bytes);
}
};
@@ -68,7 +68,7 @@ public class DataStore implements AutoCloseable {
public byte[] encode(final Doc doc) {
final byte[] rootBlockNumber = VariableByteEncoder.encode(doc.getRootBlockNumber());
final byte[] tags = doc.getTags().getFilenameBytes();
final byte[] tags = doc.getTags().toBytes();
final byte[] result = new byte[rootBlockNumber.length + tags.length];
@@ -83,7 +83,7 @@ public class DataStore implements AutoCloseable {
final long rootBlockNumber = VariableByteEncoder.decodeFirstValue(bytes);
final int bytesRootBlockNumber = VariableByteEncoder.neededBytes(rootBlockNumber);
final Tags tags = new Tags(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
final Tags tags = Tags.fromBytes(Arrays.copyOfRange(bytes, bytesRootBlockNumber, bytes.length));
return new Doc(tags, rootBlockNumber);
}
};