andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

move StringCompressor initialization to StringCompressor

Browse Source
This commit is contained in:
Andreas Huber
2021-09-18 15:39:54 +02:00
parent 6cd6f578e7
commit 6a3d711838
6 changed files with 25 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
data-store/src/main/java/org/lucares/pdb/datastore/Entry.java
View File

@@ -37,8 +37,7 @@ public class Entry {
public String toString() {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " ("
+ Tags.STRING_COMPRESSOR.asString(tags) + ")";
return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags + ")";
}
@Override

14
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -40,8 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore implements AutoCloseable {
private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
@@ -60,8 +59,6 @@ public class DataStore implements AutoCloseable {
// ids when getting them from the BSFiles)
private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis());
public static Tag TAG_ALL_DOCS = null;
private static final class PartitionedTagsCacheKey {
private final Tags tags;
private final ParititionId partitionId;
@@ -125,11 +122,6 @@ public class DataStore implements AutoCloseable {
storageBasePath = storageDirectory(dataDirectory);
Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath));
Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY);
Tags.STRING_COMPRESSOR.put("");
TAG_ALL_DOCS = Tags.STRING_COMPRESSOR.createTag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the
// StringCompressor internally, so it
// must be initialized after the string compressor has been created
diskStorage = new PartitionDiskStore(storageBasePath, "data.bs");
@@ -186,7 +178,7 @@ public class DataStore implements AutoCloseable {
// store mapping from tag to docId, so that we can find all docs for a given tag
final List<Tag> ts = new ArrayList<>(tags.toTags());
ts.add(TAG_ALL_DOCS);
ts.add(StringCompressor.TAG_ALL_DOCS);
for (final Tag tag : ts) {
Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag);
@@ -276,7 +268,7 @@ public class DataStore implements AutoCloseable {
tagToDocsId.visitValues(partitionIdSource, keyPrefix,
(tag, __) -> keys.add(Tags.STRING_COMPRESSOR.getKeyAsString(tag)));
keys.remove(ALL_DOCS_KEY);
keys.remove(StringCompressor.ALL_DOCS_KEY);
final List<String> result = new ArrayList<>(keys);
Collections.sort(result);
return result;

4
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java
View File

@@ -9,10 +9,10 @@ import java.util.stream.Collectors;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.StringCompressor;
import org.lucares.pdb.api.Tag;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.LongStreamFile;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.datastore.internal.DatePartitioner;
import org.lucares.pdb.datastore.internal.ParititionId;
import org.lucares.pdb.datastore.internal.PartitionDiskStore;
@@ -131,7 +131,7 @@ public class ExpressionToDocIdVisitor extends ExpressionVisitor<PartitionLongLis
final Set<ParititionId> availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner);
for (final ParititionId partitionId : availablePartitionIds) {
final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS);
final Long blockOffset = keyToValueToDocId.getValue(partitionId, StringCompressor.TAG_ALL_DOCS);
if (blockOffset != null) {
final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId);

24
pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java
View File

@@ -8,8 +8,13 @@ import java.util.function.Function;
*/
public class StringCompressor {
public static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use
// area
private static final String DEFAULT_GROUP = "<none>";
public static Tag TAG_ALL_DOCS;
private final UniqueStringIntegerPairs usip;
public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException {
@@ -18,7 +23,13 @@ public class StringCompressor {
public static StringCompressor create(final Path path) {
final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path);
return new StringCompressor(mapsi);
final StringCompressor result = new StringCompressor(mapsi);
result.put(ALL_DOCS_KEY);
result.put("");
TAG_ALL_DOCS = result.createTag(ALL_DOCS_KEY, "");
return result;
}
public int put(final String string) {
@@ -53,8 +64,8 @@ public class StringCompressor {
* @param value the value
*/
public Tag createTag(final String field, final String value) {
final int f = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1;
final int v = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1;
final int f = field != null ? getIfPresent(field) : -1;
final int v = value != null ? getIfPresent(value) : -1;
return new Tag(f, v);
}
@@ -126,12 +137,13 @@ public class StringCompressor {
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
for (final Tag tag : tags.toTags()) {
final String value = get(tag.getValue());
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
result.append(value);
}
}
return result.toString();
}

2
pdb-api/src/main/java/org/lucares/pdb/api/Tag.java
View File

@@ -12,7 +12,7 @@ public class Tag implements Comparable<Tag> {
/**
* Create a new tag with field and value specified as int. See
* {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints.
* {@link StringCompressor} for the mapping between Strings and ints.
*
* @param field the field as int
* @param value the value as int

13
pdb-api/src/main/java/org/lucares/pdb/api/Tags.java
View File

@@ -7,7 +7,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Function;
import org.lucares.collections.IntList;
@@ -233,16 +232,4 @@ public class Tags implements Comparable<Tags> {
public boolean isEmpty() {
return tags.isEmpty();
}
// TODO move
// to StringCompressor
public void forEach(final BiConsumer<String, String> keyValueConsumer) {
for (final Tag tag : tags) {
final String key = STRING_COMPRESSOR.get(tag.getKey());
final String value = STRING_COMPRESSOR.get(tag.getValue());
keyValueConsumer.accept(key, value);
}
}
}