From 6a3d71183895c6a2fe597f77845a37341403bb82 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 18 Sep 2021 15:39:54 +0200 Subject: [PATCH] move StringCompressor initialization to StringCompressor --- .../java/org/lucares/pdb/datastore/Entry.java | 3 +-- .../pdb/datastore/internal/DataStore.java | 14 +++-------- .../lang/ExpressionToDocIdVisitor.java | 4 ++-- .../org/lucares/pdb/api/StringCompressor.java | 24 ++++++++++++++----- .../main/java/org/lucares/pdb/api/Tag.java | 2 +- .../main/java/org/lucares/pdb/api/Tags.java | 13 ---------- 6 files changed, 25 insertions(+), 35 deletions(-) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/Entry.java b/data-store/src/main/java/org/lucares/pdb/datastore/Entry.java index 32d0d22..176cc53 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/Entry.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/Entry.java @@ -37,8 +37,7 @@ public class Entry { public String toString() { final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); - return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" - + Tags.STRING_COMPRESSOR.asString(tags) + ")"; + return date.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + " = " + value + " (" + tags + ")"; } @Override diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java index 0653a06..d0a68ec 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java @@ -40,8 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DataStore implements AutoCloseable { - private static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use - // area + private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory .getLogger("org.lucares.metrics.dataStore.executeQuery"); private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory @@ -60,8 +59,6 @@ public class DataStore implements AutoCloseable { // ids when getting them from the BSFiles) private static final AtomicLong NEXT_DOC_ID = new AtomicLong(System.currentTimeMillis()); - public static Tag TAG_ALL_DOCS = null; - private static final class PartitionedTagsCacheKey { private final Tags tags; private final ParititionId partitionId; @@ -125,11 +122,6 @@ public class DataStore implements AutoCloseable { storageBasePath = storageDirectory(dataDirectory); Tags.STRING_COMPRESSOR = StringCompressor.create(keyCompressionFile(storageBasePath)); - Tags.STRING_COMPRESSOR.put(ALL_DOCS_KEY); - Tags.STRING_COMPRESSOR.put(""); - TAG_ALL_DOCS = Tags.STRING_COMPRESSOR.createTag(ALL_DOCS_KEY, ""); // Tag(String, String) uses the - // StringCompressor internally, so it - // must be initialized after the string compressor has been created diskStorage = new PartitionDiskStore(storageBasePath, "data.bs"); @@ -186,7 +178,7 @@ public class DataStore implements AutoCloseable { // store mapping from tag to docId, so that we can find all docs for a given tag final List ts = new ArrayList<>(tags.toTags()); - ts.add(TAG_ALL_DOCS); + ts.add(StringCompressor.TAG_ALL_DOCS); for (final Tag tag : ts) { Long diskStoreOffsetForDocIdsOfTag = tagToDocsId.getValue(partitionId, tag); @@ -276,7 +268,7 @@ public class DataStore implements AutoCloseable { tagToDocsId.visitValues(partitionIdSource, keyPrefix, (tag, __) -> keys.add(Tags.STRING_COMPRESSOR.getKeyAsString(tag))); - keys.remove(ALL_DOCS_KEY); + keys.remove(StringCompressor.ALL_DOCS_KEY); final List result = new ArrayList<>(keys); Collections.sort(result); return result; diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java index 12103e0..1161394 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java @@ -9,10 +9,10 @@ import java.util.stream.Collectors; import org.lucares.collections.LongList; import org.lucares.pdb.api.DateTimeRange; +import org.lucares.pdb.api.StringCompressor; import org.lucares.pdb.api.Tag; import org.lucares.pdb.api.Tags; import org.lucares.pdb.blockstorage.LongStreamFile; -import org.lucares.pdb.datastore.internal.DataStore; import org.lucares.pdb.datastore.internal.DatePartitioner; import org.lucares.pdb.datastore.internal.ParititionId; import org.lucares.pdb.datastore.internal.PartitionDiskStore; @@ -131,7 +131,7 @@ public class ExpressionToDocIdVisitor extends ExpressionVisitor availablePartitionIds = keyToValueToDocId.getAvailablePartitionIds(datePartitioner); for (final ParititionId partitionId : availablePartitionIds) { - final Long blockOffset = keyToValueToDocId.getValue(partitionId, DataStore.TAG_ALL_DOCS); + final Long blockOffset = keyToValueToDocId.getValue(partitionId, StringCompressor.TAG_ALL_DOCS); if (blockOffset != null) { final LongStreamFile bsFile = diskStorage.streamExistingFile(blockOffset, partitionId); diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java b/pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java index f8e6572..eb91da0 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/StringCompressor.java @@ -8,8 +8,13 @@ import java.util.function.Function; */ public class StringCompressor { + public static final String ALL_DOCS_KEY = "\ue001allDocs"; // \ue001 is the second character in the private use + // area + private static final String DEFAULT_GROUP = ""; + public static Tag TAG_ALL_DOCS; + private final UniqueStringIntegerPairs usip; public StringCompressor(final UniqueStringIntegerPairs usip) throws RuntimeIOException { @@ -18,7 +23,13 @@ public class StringCompressor { public static StringCompressor create(final Path path) { final UniqueStringIntegerPairs mapsi = new UniqueStringIntegerPairs(path); - return new StringCompressor(mapsi); + final StringCompressor result = new StringCompressor(mapsi); + + result.put(ALL_DOCS_KEY); + result.put(""); + TAG_ALL_DOCS = result.createTag(ALL_DOCS_KEY, ""); + + return result; } public int put(final String string) { @@ -53,8 +64,8 @@ public class StringCompressor { * @param value the value */ public Tag createTag(final String field, final String value) { - final int f = field != null ? Tags.STRING_COMPRESSOR.getIfPresent(field) : -1; - final int v = value != null ? Tags.STRING_COMPRESSOR.getIfPresent(value) : -1; + final int f = field != null ? getIfPresent(field) : -1; + final int v = value != null ? getIfPresent(value) : -1; return new Tag(f, v); } @@ -126,12 +137,13 @@ public class StringCompressor { if (tags.isEmpty()) { result.append(DEFAULT_GROUP); } else { - tags.forEach((k, v) -> { + for (final Tag tag : tags.toTags()) { + final String value = get(tag.getValue()); if (result.length() > 0) { result.append(" / "); } - result.append(v); - }); + result.append(value); + } } return result.toString(); } diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/Tag.java b/pdb-api/src/main/java/org/lucares/pdb/api/Tag.java index 51f652c..4af168d 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/Tag.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/Tag.java @@ -12,7 +12,7 @@ public class Tag implements Comparable { /** * Create a new tag with field and value specified as int. See - * {@link Tags#STRING_COMPRESSOR} for the mapping between Strings and ints. + * {@link StringCompressor} for the mapping between Strings and ints. * * @param field the field as int * @param value the value as int diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java index 2ee21d8..13cad11 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java @@ -7,7 +7,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.TreeSet; -import java.util.function.BiConsumer; import java.util.function.Function; import org.lucares.collections.IntList; @@ -233,16 +232,4 @@ public class Tags implements Comparable { public boolean isEmpty() { return tags.isEmpty(); } - -// TODO move -// to StringCompressor - - public void forEach(final BiConsumer keyValueConsumer) { - - for (final Tag tag : tags) { - final String key = STRING_COMPRESSOR.get(tag.getKey()); - final String value = STRING_COMPRESSOR.get(tag.getValue()); - keyValueConsumer.accept(key, value); - } - } }