andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

do not compute counts when proposing all keys

Browse Source
This commit is contained in:
Andreas Huber
2017-11-18 13:03:45 +01:00
parent 14d1367c4a
commit a6a2236d18
7 changed files with 584 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -5,19 +5,19 @@ public class Proposal implements Comparable<Proposal> {
private final String proposedQuery;
private final long results;
private final boolean hasResults;
public Proposal(final String proposedTag, final String proposedQuery, final long results) {
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.results = results;
this.hasResults = hasResults;
}
public Proposal(final Proposal proposal, final long results) {
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.results = results;
this.hasResults = hasResults;
}
public String getProposedTag() {
@@ -28,35 +28,42 @@ public class Proposal implements Comparable<Proposal> {
return proposedQuery;
}
public long getResults() {
return results;
public boolean hasResults() {
return hasResults;
}
@Override
public String toString() {
return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", results=" + results
return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", hasResults=" + hasResults
+ "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
result = prime * result + (int) (results ^ (results >>> 32));
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result
+ ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result
+ ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
@@ -67,18 +74,11 @@ public class Proposal implements Comparable<Proposal> {
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
if (results != other.results)
return false;
return true;
}
@Override
public int compareTo(final Proposal o) {
if (results != o.results) {
return results < o.results ? 1 : -1;
}
return proposedTag.compareToIgnoreCase(o.proposedTag);
public int compareTo(Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
}

5
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -22,8 +22,11 @@ import org.lucares.pdb.datastore.lang.Expression;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds;
import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final String SUBDIR_STORAGE = "storage";
private static final String PDB_EXTENSION = ".pdb";
@@ -196,10 +199,12 @@ public class DataStore {
}
private IntList executeQuery(final String query) {
final long start = System.nanoTime();
final Expression expression = QueryLanguageParser.parse(query);
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(keyToValueToDocId,
new AllDocIds(docIdToDoc));
final IntList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results " , query, (System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}

12
data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java
View File

@@ -28,7 +28,7 @@ public class Proposer {
result = ProposerParser.parse(query, dataStore, caretIndex);
}
return CollectionUtils.filter(result, p -> p.getResults() >= 0);
return CollectionUtils.filter(result, p -> p.hasResults() );
}
private SortedSet<Proposal> proposeForAllKeys() {
@@ -37,23 +37,21 @@ public class Proposer {
final Map<String, String> fieldToQuery = CollectionUtils.createMapFromKeys(fields, f -> f + "=*");
result = computeProposalsForQueries(fieldToQuery);
result = toProposalsForQueries(fieldToQuery);
return result;
}
private SortedSet<Proposal> computeProposalsForQueries(final Map<String, String> keyToQuery) {
private SortedSet<Proposal> toProposalsForQueries(final Map<String, String> keyToQuery) {
final SortedSet<Proposal> result = new TreeSet<>();
for (final Entry<String, String> e : keyToQuery.entrySet()) {
final String key = e.getKey();
final String query = e.getValue();
final int count = dataStore.count(query);
final Proposal proposal = new Proposal(key, query, count);
final Proposal proposal = new Proposal(key, query, true);
result.add(proposal);
}
return result;
}
}

552
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java
View File

@@ -1,276 +1,276 @@
package org.lucares.pdb.datastore.lang;
import java.util.BitSet;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.CollectionUtils;
public class QueryCompletionPdbLangParser extends PdbLangParser {
public class Listener implements PdbLangListener, ANTLRErrorListener {
private final int caretPosition;
private final DataStore dataStore;
private final SortedSet<Proposal> proposals = new TreeSet<>();
private final String query;
public Listener(final String query, final DataStore dataStore, final int caretPosition) {
this.query = query;
this.dataStore = dataStore;
this.caretPosition = caretPosition;
}
public SortedSet<Proposal> getProposals() {
return proposals;
}
@Override
public void visitTerminal(final TerminalNode node) {
if (containsCaret(node) && !isEOF(node)) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
if (_ctx instanceof PropertyTerminalExpressionContext) {
final String propertyKey = _ctx.getParent().children.get(0).getText();
final String propertyValuePrefix = node.getText().substring(0, caretPosition - start);
final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey,
propertyValuePrefix);
proposedValues.stream()//
.map(v -> {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(start, end + 1, v + " ");
return new Proposal(v, newQuery.toString(), -1);
}).map(p -> {
final int count = dataStore.count(p.getProposedQuery());
return new Proposal(p, count);
}).forEach(proposals::add);
} else if (_ctx instanceof IdentifierExpressionContext) {
final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start);
final StringBuilder newQueryPattern = new StringBuilder(query);
newQueryPattern.replace(start, end + 1, "%s");
addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString());
}
}
}
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) {
final CommonToken token = (CommonToken) offendingSymbol;
final String text = token.getText();
if ("and".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and ");
proposals.add(new Proposal(" and ", newQuery.toString(), 1));
}
if ("or".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or ");
proposals.add(new Proposal(" or ", newQuery.toString(), 1));
}
}
}
private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) {
final List<String> availableKeys = dataStore.getAvailableFields();
final List<String> matchingKeys = CollectionUtils.filter(availableKeys,
s -> s.startsWith(propertyKeyPrefix));
matchingKeys.stream()//
.map(key -> {
return new Proposal(key, String.format(newQueryPattern, key + "=* "), -1);
}).map(p -> {
final String proposedQuery = p.getProposedQuery();
final int count = count(proposedQuery);
return new Proposal(p, count);
}).forEach(proposals::add);
}
private int count(final String proposedQuery) {
try {
return dataStore.count(proposedQuery);
} catch (final SyntaxException e) {
return -1;
}
}
private boolean isEOF(final Object offendingSymbol) {
if (offendingSymbol instanceof CommonToken) {
return ((CommonToken) offendingSymbol).getType() < 0;
}
return false;
}
@Override
public void visitErrorNode(final ErrorNode node) {
}
@Override
public void enterEveryRule(final ParserRuleContext ctx) {
}
@Override
public void exitEveryRule(final ParserRuleContext ctx) {
}
@Override
public void enterStart(final StartContext ctx) {
}
@Override
public void exitStart(final StartContext ctx) {
}
@Override
public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void enterNotExpression(final NotExpressionContext ctx) {
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
}
@Override
public void enterParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void exitParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void enterPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void enterIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
// if (containsCaret(ctx)) {
// final int start = ctx.getStart().getStartIndex();
// final int end = ctx.getStop().getStopIndex();
// final int ruleIndex = _ctx.getRuleIndex();
//
// final String prefix = ctx.getText().substring(0, caretPosition -
// start);
// ctx.getParent().children.get(0).getText();
//
// proposals.addAll(getPropertyValuesByPrefix(prefix));
// }
}
private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey,
final String propertyValuePrefix) {
final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey);
final SortedSet<String> result = new TreeSet<>();
for (final String value : availableValuesForKey) {
if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) {
result.add(value);
}
}
return result;
}
@Override
public void enterEqual(final EqualContext ctx) {
}
@Override
public void exitEqual(final EqualContext ctx) {
}
private boolean isEOF(final TerminalNode node) {
return node.getSymbol().getType() < 0;
}
private boolean containsCaret(final TerminalNode node) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
return start <= caretPosition && end + 1 >= caretPosition;
}
@Override
public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex,
final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) {
}
@Override
public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) {
}
@Override
public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final int prediction, final ATNConfigSet configs) {
}
}
public QueryCompletionPdbLangParser(final TokenStream input) {
super(input);
}
}
package org.lucares.pdb.datastore.lang;
import java.util.BitSet;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.CollectionUtils;
public class QueryCompletionPdbLangParser extends PdbLangParser {
public class Listener implements PdbLangListener, ANTLRErrorListener {
private final int caretPosition;
private final DataStore dataStore;
private final SortedSet<Proposal> proposals = new TreeSet<>();
private final String query;
public Listener(final String query, final DataStore dataStore, final int caretPosition) {
this.query = query;
this.dataStore = dataStore;
this.caretPosition = caretPosition;
}
public SortedSet<Proposal> getProposals() {
return proposals;
}
@Override
public void visitTerminal(final TerminalNode node) {
if (containsCaret(node) && !isEOF(node)) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
if (_ctx instanceof PropertyTerminalExpressionContext) {
final String propertyKey = _ctx.getParent().children.get(0).getText();
final String propertyValuePrefix = node.getText().substring(0, caretPosition - start);
final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey,
propertyValuePrefix);
proposedValues.stream()//
.map(v -> {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(start, end + 1, v + " "); // insert the terminal into the query
return new Proposal(v, newQuery.toString(), false);
}).map(p -> {
final int count = dataStore.count(p.getProposedQuery());
return new Proposal(p, count> 0);
}).forEach(proposals::add);
} else if (_ctx instanceof IdentifierExpressionContext) {
final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start);
final StringBuilder newQueryPattern = new StringBuilder(query);
newQueryPattern.replace(start, end + 1, "%s");
addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString());
}
}
}
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) {
final CommonToken token = (CommonToken) offendingSymbol;
final String text = token.getText();
if ("and".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and ");
proposals.add(new Proposal(" and ", newQuery.toString(), true));
}
if ("or".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or ");
proposals.add(new Proposal(" or ", newQuery.toString(), true));
}
}
}
private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) {
final List<String> availableKeys = dataStore.getAvailableFields();
final List<String> matchingKeys = CollectionUtils.filter(availableKeys,
s -> s.startsWith(propertyKeyPrefix));
matchingKeys.stream()//
.map(key -> {
return new Proposal(key, String.format(newQueryPattern, key + "=* "), false);
}).map(p -> {
final String proposedQuery = p.getProposedQuery();
final int count = count(proposedQuery);
return new Proposal(p, count > 0);
}).forEach(proposals::add);
}
private int count(final String proposedQuery) {
try {
return dataStore.count(proposedQuery);
} catch (final SyntaxException e) {
return -1;
}
}
private boolean isEOF(final Object offendingSymbol) {
if (offendingSymbol instanceof CommonToken) {
return ((CommonToken) offendingSymbol).getType() < 0;
}
return false;
}
@Override
public void visitErrorNode(final ErrorNode node) {
}
@Override
public void enterEveryRule(final ParserRuleContext ctx) {
}
@Override
public void exitEveryRule(final ParserRuleContext ctx) {
}
@Override
public void enterStart(final StartContext ctx) {
}
@Override
public void exitStart(final StartContext ctx) {
}
@Override
public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void enterNotExpression(final NotExpressionContext ctx) {
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
}
@Override
public void enterParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void exitParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void enterPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void enterIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
// if (containsCaret(ctx)) {
// final int start = ctx.getStart().getStartIndex();
// final int end = ctx.getStop().getStopIndex();
// final int ruleIndex = _ctx.getRuleIndex();
//
// final String prefix = ctx.getText().substring(0, caretPosition -
// start);
// ctx.getParent().children.get(0).getText();
//
// proposals.addAll(getPropertyValuesByPrefix(prefix));
// }
}
private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey,
final String propertyValuePrefix) {
final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey);
final SortedSet<String> result = new TreeSet<>();
for (final String value : availableValuesForKey) {
if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) {
result.add(value);
}
}
return result;
}
@Override
public void enterEqual(final EqualContext ctx) {
}
@Override
public void exitEqual(final EqualContext ctx) {
}
private boolean isEOF(final TerminalNode node) {
return node.getSymbol().getType() < 0;
}
private boolean containsCaret(final TerminalNode node) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
return start <= caretPosition && end + 1 >= caretPosition;
}
@Override
public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex,
final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) {
}
@Override
public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) {
}
@Override
public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final int prediction, final ATNConfigSet configs) {
}
}
public QueryCompletionPdbLangParser(final TokenStream input) {
super(input);
}
}