From b5e2d0a217e155f3717a0f5f4c2800185a244d0b Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 16 Mar 2019 10:19:28 +0100 Subject: [PATCH] introduce clustering for query completion indices --- .../pdb/datastore/internal/ClusterId.java | 4 + .../internal/ClusteredPersistentMap.java | 18 ++- .../pdb/datastore/internal/DataStore.java | 7 +- .../internal/QueryCompletionIndex.java | 147 ++++++++---------- .../lang/FindValuesForQueryCompletion.java | 32 ++-- .../pdb/datastore/lang/NewProposerParser.java | 46 +++--- .../pdb/datastore/internal/DataStoreTest.java | 10 +- .../pdb/datastore/internal/ProposerTest.java | 8 +- .../internal/QueryCompletionIndexTest.java | 16 +- .../org/lucares/pdb/api/DateTimeRange.java | 7 +- .../org/lucares/pdb/api/QueryConstants.java | 5 + .../lucares/pdb/api/QueryWithCaretMarker.java | 18 +++ .../java/org/lucares/pdbui/PdbController.java | 10 +- .../lucares/performance/db/PerformanceDb.java | 5 +- 14 files changed, 197 insertions(+), 136 deletions(-) create mode 100644 pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java create mode 100644 pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusterId.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusterId.java index cb034dd..4e08cff 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusterId.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusterId.java @@ -14,6 +14,10 @@ public class ClusterId { this.clusterId = clusterId; } + public static ClusterId of(final String clusterId) { + return new ClusterId(clusterId); + } + /** * @return the id, e.g. a time like 201902 (cluster for entries of February * 2019) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusteredPersistentMap.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusteredPersistentMap.java index a58018b..ba67a97 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusteredPersistentMap.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/ClusteredPersistentMap.java @@ -45,10 +45,18 @@ public class ClusteredPersistentMap implements AutoCloseable { }; } + private PersistentMap getExistingPersistentMap(final ClusterId clusterId) { + return maps.computeIfAbsent(clusterId, supplier); + } + + private PersistentMap getPersistentMapCreateIfNotExists(final ClusterId clusterId) { + return maps.computeIfAbsent(clusterId, creator); + } + public V getValue(final ClusterId clusterId, final K key) { try { - final PersistentMap map = maps.computeIfAbsent(clusterId, supplier); + final PersistentMap map = getExistingPersistentMap(clusterId); return map != null ? map.getValue(key) : null; } catch (final IOException e) { throw new ReadRuntimeException(e); @@ -61,7 +69,7 @@ public class ClusteredPersistentMap implements AutoCloseable { final List clusterIds = clusterIdSource.toClusterIds(); for (final ClusterId clusterId : clusterIds) { - final PersistentMap map = maps.computeIfAbsent(clusterId, creator); + final PersistentMap map = getPersistentMapCreateIfNotExists(clusterId); if (map != null) { final V value = map.getValue(key); if (value != null) { @@ -79,7 +87,7 @@ public class ClusteredPersistentMap implements AutoCloseable { public V putValue(final ClusterId clusterId, final K key, final V value) { try { - final PersistentMap map = maps.computeIfAbsent(clusterId, creator); + final PersistentMap map = getPersistentMapCreateIfNotExists(clusterId); return map.putValue(key, value); } catch (final IOException e) { throw new ReadRuntimeException(e); @@ -88,7 +96,7 @@ public class ClusteredPersistentMap implements AutoCloseable { public void visitValues(final ClusterId clusterId, final K keyPrefix, final Visitor visitor) { try { - final PersistentMap map = maps.computeIfAbsent(clusterId, creator); + final PersistentMap map = getExistingPersistentMap(clusterId); if (map != null) { map.visitValues(keyPrefix, visitor); } @@ -102,7 +110,7 @@ public class ClusteredPersistentMap implements AutoCloseable { final List clusterIds = clusterIdSource.toClusterIds(); for (final ClusterId clusterId : clusterIds) { - final PersistentMap map = maps.get(clusterId); + final PersistentMap map = getExistingPersistentMap(clusterId); if (map != null) { map.visitValues(keyPrefix, visitor); } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java index fd7aa22..765d6b7 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java @@ -18,6 +18,7 @@ import java.util.function.Consumer; import org.lucares.collections.LongList; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Query; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.api.RuntimeIOException; import org.lucares.pdb.api.StringCompressor; import org.lucares.pdb.api.Tag; @@ -159,7 +160,7 @@ public class DataStore implements AutoCloseable { // index the tags, so that we can efficiently find all possible values for a // field in a query - queryCompletionIndex.addTags(tags); + queryCompletionIndex.addTags(clusterId, tags); return newFilesRootBlockOffset; } catch (final IOException e) { @@ -343,10 +344,10 @@ public class DataStore implements AutoCloseable { }); } - public List propose(final String query, final int caretIndex) { + public List propose(final QueryWithCaretMarker query) { final NewProposerParser newProposerParser = new NewProposerParser(queryCompletionIndex); - final List proposals = newProposerParser.propose(query, caretIndex); + final List proposals = newProposerParser.propose(query); LOGGER.debug("Proposals for query {}: {}", query, proposals); return proposals; } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java index 19d51a5..e2a0b28 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java @@ -7,7 +7,7 @@ import java.util.SortedSet; import java.util.TreeSet; import org.lucares.collections.LongList; -import org.lucares.pdb.api.RuntimeIOException; +import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Tag; import org.lucares.pdb.api.Tags; import org.lucares.pdb.map.Empty; @@ -208,22 +208,22 @@ public class QueryCompletionIndex implements AutoCloseable { } } - private final PersistentMap tagToTagIndex; - private final PersistentMap fieldToValueIndex; - private final PersistentMap fieldIndex; + private final ClusteredPersistentMap tagToTagIndex; + private final ClusteredPersistentMap fieldToValueIndex; + private final ClusteredPersistentMap fieldIndex; public QueryCompletionIndex(final Path basePath) throws IOException { - final Path tagToTagIndexFile = basePath.resolve("queryCompletionTagToTagIndex.bs"); - tagToTagIndex = new PersistentMap<>(tagToTagIndexFile, new EncoderTwoTags(), PersistentMap.EMPTY_ENCODER); + tagToTagIndex = new ClusteredPersistentMap<>(basePath, "queryCompletionTagToTagIndex.bs", new EncoderTwoTags(), + PersistentMap.EMPTY_ENCODER); - final Path fieldToValueIndexFile = basePath.resolve("queryCompletionFieldToValueIndex.bs"); - fieldToValueIndex = new PersistentMap<>(fieldToValueIndexFile, new EncoderTag(), PersistentMap.EMPTY_ENCODER); + fieldToValueIndex = new ClusteredPersistentMap<>(basePath, "queryCompletionFieldToValueIndex.bs", + new EncoderTag(), PersistentMap.EMPTY_ENCODER); - final Path fieldIndexFile = basePath.resolve("queryCompletionFieldIndex.bs"); - fieldIndex = new PersistentMap<>(fieldIndexFile, new EncoderField(), PersistentMap.EMPTY_ENCODER); + fieldIndex = new ClusteredPersistentMap<>(basePath, "queryCompletionFieldIndex.bs", new EncoderField(), + PersistentMap.EMPTY_ENCODER); } - public void addTags(final Tags tags) throws IOException { + public void addTags(final ClusterId clusterId, final Tags tags) throws IOException { final List listOfTagsA = tags.toTags(); final List listOfTagsB = tags.toTags(); @@ -231,14 +231,14 @@ public class QueryCompletionIndex implements AutoCloseable { for (final Tag tagA : listOfTagsA) { for (final Tag tagB : listOfTagsB) { final TwoTags key = new TwoTags(tagA, tagB); - tagToTagIndex.putValue(key, Empty.INSTANCE); + tagToTagIndex.putValue(clusterId, key, Empty.INSTANCE); } } // create indices of all tags and all fields for (final Tag tag : listOfTagsA) { - fieldToValueIndex.putValue(tag, Empty.INSTANCE); - fieldIndex.putValue(tag.getKeyAsString(), Empty.INSTANCE); + fieldToValueIndex.putValue(clusterId, tag, Empty.INSTANCE); + fieldIndex.putValue(clusterId, tag.getKeyAsString(), Empty.INSTANCE); } } @@ -247,76 +247,67 @@ public class QueryCompletionIndex implements AutoCloseable { tagToTagIndex.close(); } - public SortedSet find(final String property, final String value, final String field) { + public SortedSet find(final DateTimeRange dateRange, final String property, final String value, + final String field) { final Tag tag = new Tag(property, value); - return find(tag, field); + return find(dateRange, tag, field); } - public SortedSet find(final Tag tag, final String field) { - try { - final SortedSet result = new TreeSet<>(); - final int tagBKey = Tags.STRING_COMPRESSOR.put(field); - final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See - // EncoderTwoTags - final TwoTags keyPrefix = new TwoTags(tag, tagB); - tagToTagIndex.visitValues(keyPrefix, (k, v) -> { + public SortedSet find(final DateTimeRange dateRange, final Tag tag, final String field) { + + final SortedSet result = new TreeSet<>(); + final int tagBKey = Tags.STRING_COMPRESSOR.put(field); + final Tag tagB = new Tag(tagBKey, -1); // the value must be negative for the prefix search to work. See + // EncoderTwoTags + final TwoTags keyPrefix = new TwoTags(tag, tagB); + + final ClusterIdSource clusterIdSource = new DateCluster(dateRange); + tagToTagIndex.visitValues(clusterIdSource, keyPrefix, (k, v) -> { + result.add(k.getTagB().getValueAsString()); + }); + + return result; + } + + public SortedSet findAllValuesForField(final DateTimeRange dateRange, final String field) { + + final SortedSet result = new TreeSet<>(); + final int tagKey = Tags.STRING_COMPRESSOR.put(field); + final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See + + final ClusterIdSource clusterIdSource = new DateCluster(dateRange); + fieldToValueIndex.visitValues(clusterIdSource, keyPrefix, (k, v) -> { + result.add(k.getValueAsString()); + }); + + return result; + } + + public SortedSet findAllValuesNotForField(final DateTimeRange dateRange, final Tag tag, + final String field) { + final SortedSet result = new TreeSet<>(); + + final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null); + + final int negatedValueA = tag.getValue(); + final ClusterIdSource clusterIdSource = new DateCluster(dateRange); + tagToTagIndex.visitValues(clusterIdSource, keyPrefix, (k, v) -> { + + final int valueA = k.getTagA().getValue(); + if (valueA != negatedValueA) { result.add(k.getTagB().getValueAsString()); - }); + } + }); - return result; - } catch (final IOException e) { - throw new RuntimeIOException(e); - } + return result; } - public SortedSet findAllValuesForField(final String field) { - try { - final SortedSet result = new TreeSet<>(); - final int tagKey = Tags.STRING_COMPRESSOR.put(field); - final Tag keyPrefix = new Tag(tagKey, -1); // the value must be negative for the prefix search to work. See - - fieldToValueIndex.visitValues(keyPrefix, (k, v) -> { - result.add(k.getValueAsString()); - }); - - return result; - } catch (final IOException e) { - throw new RuntimeIOException(e); - } + public SortedSet findAllFields(final DateTimeRange dateRange) { + final SortedSet result = new TreeSet<>(); + final ClusterIdSource clusterIdSource = new DateCluster(dateRange); + fieldIndex.visitValues(clusterIdSource, "", (k, v) -> { + result.add(k); + }); + return result; } - - public SortedSet findAllValuesNotForField(final Tag tag, final String field) { - try { - final SortedSet result = new TreeSet<>(); - - final TwoTags keyPrefix = new TwoTags(field, tag.getKeyAsString(), null, null); - - final int negatedValueA = tag.getValue(); - - tagToTagIndex.visitValues(keyPrefix, (k, v) -> { - - final int valueA = k.getTagA().getValue(); - if (valueA != negatedValueA) { - result.add(k.getTagB().getValueAsString()); - } - }); - - return result; - } catch (final IOException e) { - throw new RuntimeIOException(e); - } - } - - public SortedSet findAllFields() { - try { - final SortedSet result = new TreeSet<>(); - fieldIndex.visitValues("", (k, v) -> { - result.add(k); - }); - return result; - } catch (final IOException e) { - throw new RuntimeIOException(e); - } - } - } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java index 3b5f447..098d880 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/lang/FindValuesForQueryCompletion.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; +import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Tag; import org.lucares.pdb.datastore.internal.QueryCompletionIndex; import org.lucares.pdb.datastore.lang.Expression.And; @@ -29,8 +30,11 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor> { private final QueryCompletionIndex index; private final String field; + private final DateTimeRange dateTimeRange; - public AndCaretExpressionVisitor(final QueryCompletionIndex queryCompletionIndex, final String field) { + public AndCaretExpressionVisitor(final DateTimeRange dateTimeRange, + final QueryCompletionIndex queryCompletionIndex, final String field) { + this.dateTimeRange = dateTimeRange; index = queryCompletionIndex; this.field = field; } @@ -41,7 +45,7 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor result = index.find(fieldA, valueA, field); + final SortedSet result = index.find(dateTimeRange, fieldA, valueA, field); METRIC_AND_CARET_LOGGER.debug("{}: {}ms", property, (System.nanoTime() - start) / 1_000_000.0); return result; } @@ -53,7 +57,7 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor values = expression.getValues(); for (final String value : values) { - final SortedSet candidates = index.find(property, value, field); + final SortedSet candidates = index.find(dateTimeRange, property, value, field); result.addAll(candidates); } METRIC_AND_CARET_LOGGER.debug("{}: {}ms", expression, (System.nanoTime() - start) / 1_000_000.0); @@ -119,8 +123,8 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor valuesNotForField = index.findAllValuesNotForField(tag, field); - final SortedSet valuesForField = index.find(tag, field); + final SortedSet valuesNotForField = index.findAllValuesNotForField(dateTimeRange, tag, field); + final SortedSet valuesForField = index.find(dateTimeRange, tag, field); final SortedSet valuesOnlyAvailableInField = CollectionUtils.removeAll(valuesForField, valuesNotForField, TreeSet::new); @@ -134,7 +138,11 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor allValuesForField = queryCompletionIndex.findAllValuesForField(field); + final SortedSet allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field); final String valuePrefix = value.substring(0, value.indexOf(NewProposerParser.CARET_MARKER)); @@ -167,7 +175,7 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor candidateValues = rightHandExpression - .visit(new AndCaretExpressionVisitor(queryCompletionIndex, field)); + .visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field)); final TreeSet result = GloblikePattern.filterValues(candidateValues, valuePrefix, TreeSet::new); @@ -185,15 +193,15 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor allValuesForField = queryCompletionIndex - .findAllValuesForField(caretExpression.getProperty()); + final SortedSet allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, + caretExpression.getProperty()); final SortedSet valuesForFieldMatchingCaretExpression = GloblikePattern.filterValues(allValuesForField, valuePattern, TreeSet::new); final Expression rightHandExpression = expression.getExpression(); final SortedSet rightHandValues = rightHandExpression - .visit(new AndCaretExpressionVisitor(queryCompletionIndex, field)); + .visit(new AndCaretExpressionVisitor(dateRange, queryCompletionIndex, field)); if (rightHandValues.size() == 1) { // there is only one alternative and that one must not be chosen @@ -213,7 +221,7 @@ public class FindValuesForQueryCompletion extends ExpressionVisitor allValuesForField = queryCompletionIndex.findAllValuesForField(field); + final SortedSet allValuesForField = queryCompletionIndex.findAllValuesForField(dateRange, field); final String valueWithCaretMarker = ((Property) innerExpression).getValue().getValue(); final String valuePrefix = valueWithCaretMarker.substring(0, valueWithCaretMarker.indexOf(NewProposerParser.CARET_MARKER)); diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java index a87b956..799296c 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java @@ -9,20 +9,21 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.lucares.pdb.api.DateTimeRange; +import org.lucares.pdb.api.QueryConstants; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.internal.QueryCompletionIndex; import org.lucares.utils.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class NewProposerParser { +public class NewProposerParser implements QueryConstants { private static final Logger LOGGER = LoggerFactory.getLogger(NewProposerParser.class); private final static Logger METRICS_LOGGER_PROPOSE = LoggerFactory.getLogger("org.lucares.metrics.propose"); - public final static String CARET_MARKER = "\ue001"; // character in the private use area - /* * Regex matching a java identifier without a caret marker. We define it as a * blacklist, because this is easer. The regex is only used after the @@ -36,16 +37,16 @@ public class NewProposerParser { this.queryCompletionIndex = queryCompletionIndex; } - public List propose(final String query, final int caretIndex) { + public List propose(final QueryWithCaretMarker query) { final long start = System.nanoTime(); List proposals; - if (StringUtils.isBlank(query)) { - proposals = proposeForAllKeys(); + if (StringUtils.isBlank(query.getQuery())) { + proposals = proposeForAllKeys(query.getDateRange()); } else { - final List foundProposals = proposalsForValues(query, caretIndex); + final List foundProposals = proposalsForValues(query); if (foundProposals.isEmpty()) { - proposals = proposalsForNonValues(query, caretIndex); + proposals = proposalsForNonValues(query); } else { proposals = foundProposals; } @@ -56,7 +57,7 @@ public class NewProposerParser { return proposals; } - private List proposalsForNonValues(final String query, final int caretIndex) { + private List proposalsForNonValues(final QueryWithCaretMarker query) { final List proposals = new ArrayList<>(); /* @@ -66,9 +67,7 @@ public class NewProposerParser { * location in the query (not at the caret position). */ - final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, CARET_MARKER).toString(); - - final List tokens = QueryLanguage.getTokens(queryWithCaretMarker); + final List tokens = QueryLanguage.getTokens(query.getQueryWithCaretMarker()); final int indexTokenWithCaret = CollectionUtils.indexOf(tokens, t -> t.contains(CARET_MARKER)); if (indexTokenWithCaret > 0) { @@ -78,7 +77,7 @@ public class NewProposerParser { case "and": case "or": case "!": - proposals.addAll(proposeForAllKeys(queryWithCaretMarker)); + proposals.addAll(proposeForAllKeys(query)); break; case ")": @@ -87,24 +86,25 @@ public class NewProposerParser { break; } } else if (indexTokenWithCaret == 0) { - proposals.addAll(proposeForAllKeys(queryWithCaretMarker)); + proposals.addAll(proposeForAllKeys(query)); } return proposals; } - private Collection proposeForAllKeys(final String queryWithCaretMarker) { + private Collection proposeForAllKeys(final QueryWithCaretMarker query) { final List proposals = new ArrayList<>(); - final String wordPrefix = wordPrefix(queryWithCaretMarker); + final String wordPrefix = wordPrefix(query.getQueryWithCaretMarker()); if (wordPrefix != null) { - final SortedSet allFields = queryCompletionIndex.findAllFields(); + final SortedSet allFields = queryCompletionIndex.findAllFields(query.getDateRange()); for (final String field : allFields) { if (!field.startsWith(wordPrefix)) { continue; } + final String queryWithCaretMarker = query.getQueryWithCaretMarker(); final String proposedQuery = queryWithCaretMarker .replaceAll(REGEX_IDENTIFIER + CARET_MARKER + REGEX_IDENTIFIER, field + "=* "); final String newQueryWithCaretMarker = queryWithCaretMarker @@ -131,10 +131,10 @@ public class NewProposerParser { return null; } - private List proposeForAllKeys() { + private List proposeForAllKeys(final DateTimeRange dateRange) { final List proposals = new ArrayList<>(); - final SortedSet allFields = queryCompletionIndex.findAllFields(); + final SortedSet allFields = queryCompletionIndex.findAllFields(dateRange); for (final String field : allFields) { final String proposedQuery = field + "=*"; final String newQuery = field + "="; @@ -146,26 +146,26 @@ public class NewProposerParser { return proposals; } - List proposalsForValues(final String query, final int caretIndex) { + List proposalsForValues(final QueryWithCaretMarker query) { try { // Add caret marker, so that we know where the caret is. // This also makes sure that a query like "name=|" ('|' is the caret) can be // parsed. // Without the caret marker the query would be "name=", which is not a valid // expression. - final String queryWithCaretMarker = new StringBuilder(query).insert(caretIndex, CARET_MARKER).toString(); + final String queryWithCaretMarker = query.getQueryWithCaretMarker(); // parse the query final Expression expression = QueryLanguageParser.parse(queryWithCaretMarker); - // normalize it, so that we can use the queryCompletionIndex to search vor + // normalize it, so that we can use the queryCompletionIndex to search for // candidate values final QueryCompletionExpressionOptimizer optimizer = new QueryCompletionExpressionOptimizer(); final Expression normalizedExpression = optimizer.normalizeExpression(expression); // find all candidate values final SortedSet candidateValues = normalizedExpression - .visit(new FindValuesForQueryCompletion(queryCompletionIndex)); + .visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex)); final SortedSet candidateValuesCutAtDots = cutAtDots(candidateValues, queryWithCaretMarker); diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java index 3ab1141..088762a 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java @@ -25,6 +25,7 @@ import javax.swing.JTextField; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Query; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.api.Tags; import org.lucares.pdb.blockstorage.BSFile; import org.lucares.pdb.datastore.Doc; @@ -230,7 +231,8 @@ public class DataStoreTest { Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"), Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John")); - final ClusterId clusterId = DateIndexExtension.now(); + final DateTimeRange dateRange = DateTimeRange.relativeMillis(0); + final ClusterId clusterId = DateIndexExtension.toClusterIds(dateRange).get(0); tags.forEach(t -> dataStore.createNewFile(clusterId, t)); final JFrame frame = new JFrame(); @@ -251,7 +253,9 @@ public class DataStoreTest { final String query = input.getText(); final int caretIndex = input.getCaretPosition(); - final List proposals = dataStore.propose(query, caretIndex); + final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex); + + final List proposals = dataStore.propose(q); final StringBuilder out = new StringBuilder(); @@ -291,7 +295,7 @@ public class DataStoreTest { final List expectedProposedValues) { final String query = queryWithCaret.replace("|", ""); final int caretIndex = queryWithCaret.indexOf("|"); - final List proposals = dataStore.propose(query, caretIndex); + final List proposals = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex)); System.out.println( "proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList())); diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java index 266e296..ec4b7c6 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java @@ -7,6 +7,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.lucares.pdb.api.DateTimeRange; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.api.Tags; import org.lucares.pdb.datastore.Proposal; import org.lucares.utils.CollectionUtils; @@ -21,6 +23,7 @@ public class ProposerTest { private Path dataDirectory; private DataStore dataStore; + private DateTimeRange dateRange; @BeforeClass public void beforeClass() throws Exception { @@ -38,7 +41,8 @@ public class ProposerTest { private void initDatabase() throws Exception { dataStore = new DataStore(dataDirectory); - final ClusterId now = DateIndexExtension.now(); + dateRange = DateTimeRange.now(); + final ClusterId now = DateIndexExtension.toClusterIds(dateRange).get(0); final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim"); final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy"); @@ -158,7 +162,7 @@ public class ProposerTest { private void assertProposals(final String query, final int caretIndex, final Proposal... expected) throws InterruptedException { - final List actual = dataStore.propose(query, caretIndex); + final List actual = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex)); final List expectedList = Arrays.asList(expected); Collections.sort(expectedList); diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java index 3cf4d21..f5e25c9 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.List; import java.util.SortedSet; +import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.StringCompressor; import org.lucares.pdb.api.Tag; import org.lucares.pdb.api.Tags; @@ -41,26 +42,31 @@ public class QueryCompletionIndexTest { Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C ); + final DateTimeRange dateRange = DateTimeRange.relativeMillis(1); + final ClusterId clusterId = DateIndexExtension.toClusterIds(dateRange).get(0); + try (QueryCompletionIndex index = new QueryCompletionIndex(dataDirectory)) { for (final Tags t : tags) { - index.addTags(t); + index.addTags(clusterId, t); } // all firstnames where lastname=Doe are returned sorted alphabetically. // tags A and B match - final SortedSet firstnamesWithLastnameDoe = index.find(new Tag("lastname", "Doe"), "firstname"); + final SortedSet firstnamesWithLastnameDoe = index.find(dateRange, new Tag("lastname", "Doe"), + "firstname"); Assert.assertEquals(firstnamesWithLastnameDoe, Arrays.asList("Jane", "John")); // no duplicates are returned: // tags A and C match firstname=John, but both have country=Atlantis - final SortedSet countryWithFirstnameJohn = index.find(new Tag("firstname", "John"), "country"); + final SortedSet countryWithFirstnameJohn = index.find(dateRange, new Tag("firstname", "John"), + "country"); Assert.assertEquals(countryWithFirstnameJohn, Arrays.asList("Atlantis")); // findAllValuesForField sorts alphabetically - final SortedSet firstnames = index.findAllValuesForField("firstname"); + final SortedSet firstnames = index.findAllValuesForField(dateRange, "firstname"); Assert.assertEquals(firstnames, Arrays.asList("Jane", "John"), "found: " + firstnames); - final SortedSet countries = index.findAllValuesForField("country"); + final SortedSet countries = index.findAllValuesForField(dateRange, "country"); Assert.assertEquals(countries, Arrays.asList("Atlantis", "ElDorado")); } } diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java b/pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java index 72dbb4d..e92d745 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/DateTimeRange.java @@ -15,8 +15,13 @@ public class DateTimeRange { this.end = end; } + public static DateTimeRange now() { + return relativeMillis(0); + } + public static DateTimeRange relative(final long amount, final TemporalUnit unit) { - return new DateTimeRange(OffsetDateTime.now().minus(amount, unit), OffsetDateTime.now()); + final OffsetDateTime now = OffsetDateTime.now(); + return new DateTimeRange(now.minus(amount, unit), now); } public static DateTimeRange relativeMillis(final long amount) { diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java b/pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java new file mode 100644 index 0000000..ccdcac5 --- /dev/null +++ b/pdb-api/src/main/java/org/lucares/pdb/api/QueryConstants.java @@ -0,0 +1,5 @@ +package org.lucares.pdb.api; + +public interface QueryConstants { + String CARET_MARKER = "\ue001"; // character in the private use area +} diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java b/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java new file mode 100644 index 0000000..55df92f --- /dev/null +++ b/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java @@ -0,0 +1,18 @@ +package org.lucares.pdb.api; + +public class QueryWithCaretMarker extends Query implements QueryConstants { + + private final int caretIndex; + + public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex) { + super(query, dateRange); + this.caretIndex = caretIndex; + } + + public String getQueryWithCaretMarker() { + final StringBuilder queryBuilder = new StringBuilder(getQuery()); + final StringBuilder queryWithCaretMarker = queryBuilder.insert(caretIndex, CARET_MARKER); + return queryWithCaretMarker.toString(); + } + +} diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 94fe5ea..5659c96 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -17,6 +17,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Query; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.plot.api.AxisScale; import org.lucares.pdb.plot.api.Limit; @@ -210,10 +211,15 @@ public class PdbController implements HardcodedValues, PropertyKeys { AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query, @RequestParam(name = "caretIndex") final int caretIndex) { - final AutocompleteResponse result = new AutocompleteResponse(); + // TODO get date range from UI + // TODO time range must not be static + final DateTimeRange dateRange = DateTimeRange.relativeYears(5); final int zeroBasedCaretIndex = caretIndex - 1; + final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, zeroBasedCaretIndex); - final List proposals = db.autocomplete(query, zeroBasedCaretIndex); + final AutocompleteResponse result = new AutocompleteResponse(); + + final List proposals = db.autocomplete(q); final List nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults()); final List autocompleteProposals = toAutocompleteProposals(nonEmptyProposals); diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java index c932500..e7146a6 100644 --- a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -17,6 +17,7 @@ import org.lucares.pdb.api.Entries; import org.lucares.pdb.api.Entry; import org.lucares.pdb.api.GroupResult; import org.lucares.pdb.api.Query; +import org.lucares.pdb.api.QueryWithCaretMarker; import org.lucares.pdb.api.Result; import org.lucares.pdb.api.Tags; import org.lucares.pdb.datastore.InvalidValueException; @@ -168,9 +169,9 @@ public class PerformanceDb implements AutoCloseable { } } - public List autocomplete(final String query, final int caretIndex) { + public List autocomplete(final QueryWithCaretMarker query) { - return dataStore.propose(query, caretIndex); + return dataStore.propose(query); } public List getFields(final DateTimeRange dateRange) {