diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java deleted file mode 100644 index 10cf44c..0000000 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.lucares.pdb.datastore.internal; - -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.commons.lang3.StringUtils; -import org.lucares.pdb.datastore.Proposal; -import org.lucares.pdb.datastore.lang.ProposerParser; -import org.lucares.utils.CollectionUtils; - -public class Proposer { - - /** - * We are using an AntLR parser to find the right proposals. But that approach - * does not work if a terminal is empty. The parser does not recognize missing - * terminals. - *

- * The hack-around is to add a marker character that helps the parser to find - * the terminal. - */ - public static final String PREFIX_MARKER = "\ue001"; // second character in the private use area - - private final DataStore dataStore; - - public Proposer(final DataStore dataStore) { - this.dataStore = dataStore; - } - - public List propose(final String query, final int caretIndex) { - final SortedSet result; - - if (StringUtils.isBlank(query)) { - result = proposeForAllKeys(); - } else { - - final StringBuilder q = new StringBuilder(query); - q.insert(caretIndex, PREFIX_MARKER); - - result = ProposerParser.parse(q.toString(), dataStore, caretIndex + 1); - } - - return CollectionUtils.filter(result, Proposal::hasResults); - } - - private SortedSet proposeForAllKeys() { - final SortedSet result = new TreeSet<>(); - final List fields = dataStore.getAvailableFields(); - - for (final String field : fields) { - final String newQuery = field + "="; - final Proposal proposal = new Proposal(field, field + "=*", true, newQuery, newQuery.length()); - result.add(proposal); - } - - return result; - } -} diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/ProposerParser.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/ProposerParser.java deleted file mode 100644 index 7e4131f..0000000 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/ProposerParser.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.lucares.pdb.datastore.lang; - -import java.util.SortedSet; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.lucares.pdb.datastore.Proposal; -import org.lucares.pdb.datastore.internal.DataStore; -import org.lucares.pdb.datastore.lang.QueryCompletionPdbLangParser.Listener; - -public class ProposerParser { - - public static SortedSet parse(final String query, final DataStore dataStore, final int caretIndex) { - - final ProposerParser lang = new ProposerParser(); - return lang.parseInternal(query, dataStore, caretIndex); - } - - private SortedSet parseInternal(final String query, final DataStore dataStore, final int caretIndex) { - - final CharStream in = CharStreams.fromString(query); - - final PdbLangLexer lexer = new PdbLangLexer(in); - - final CommonTokenStream tokens = new CommonTokenStream(lexer); - - final QueryCompletionPdbLangParser parser = new QueryCompletionPdbLangParser(tokens); - parser.setTrace(false); - - final Listener listener = parser.new Listener(query, dataStore, caretIndex); - parser.addErrorListener(listener); - parser.addParseListener(listener); - - parser.start(); - - return listener.getProposals(); - } -} diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java deleted file mode 100644 index 352fce4..0000000 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java +++ /dev/null @@ -1,329 +0,0 @@ -package org.lucares.pdb.datastore.lang; - -import java.util.BitSet; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ErrorNode; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.lucares.pdb.datastore.Proposal; -import org.lucares.pdb.datastore.internal.DataStore; -import org.lucares.pdb.datastore.internal.Proposer; -import org.lucares.utils.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class QueryCompletionPdbLangParser extends PdbLangParser { - - private final static String CARET_MARKER = "\ue002"; // third character in the private use area - - private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.autocomplete"); - - public class Listener implements PdbLangListener, ANTLRErrorListener { - - private final int caretPosition; - private final DataStore dataStore; - private final SortedSet proposals = Collections.synchronizedSortedSet(new TreeSet<>()); - private final String query; - - public Listener(final String query, final DataStore dataStore, final int caretPosition) { - this.query = query; - this.dataStore = dataStore; - this.caretPosition = caretPosition; - } - - public SortedSet getProposals() { - return proposals; - } - - @Override - public void visitTerminal(final TerminalNode node) { - if (containsCaret(node) && !isEOF(node)) { - final int start = node.getSymbol().getStartIndex(); - final int end = node.getSymbol().getStopIndex(); - - if (_ctx instanceof PropertyTerminalExpressionContext) { - - final String postfixAfterInsertedTerminal; - final String propertyKey; - if (_ctx.getParent() instanceof ListOfPropValuesContext) { - // for in-expressions, e.g. key in (val) - ParserRuleContext parent = _ctx.getParent(); - while (parent instanceof ListOfPropValuesContext - || parent instanceof EnclosedListOfPropValuesContext) { - parent = parent.getParent(); - } - - propertyKey = parent.children.get(0).getText(); - postfixAfterInsertedTerminal = ""; - } else { - // for property-expressions, e.g. key = val - propertyKey = _ctx.getParent().children.get(0).getText(); - postfixAfterInsertedTerminal = " "; - } - - String propertyValuePrefix = node.getText().substring(0, caretPosition - start); - propertyValuePrefix = propertyValuePrefix.replace(Proposer.PREFIX_MARKER, ""); - final SortedSet proposedValues = getPropertyValuesByPrefix(propertyKey, - propertyValuePrefix); - - final long startTime = System.nanoTime(); - proposedValues.stream()// - .map(v -> { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(start, end + 1, v + postfixAfterInsertedTerminal); // insert the - // terminal into the - // query - - return new Proposal(v, newQuery.toString(), false, newQuery.toString(), - start + v.length() + postfixAfterInsertedTerminal.length()); - }).map(p -> { - int count = 0; - try { - count = dataStore.count(p.getProposedQuery()); - } catch (final SyntaxException e) { - // ignore: if the query is not valid, then it does not find any results - } - return new Proposal(p, count > 0); - }).forEach(proposals::add); - - METRICS_LOGGER.debug("proposals for property value {} took {} ms", propertyValuePrefix, - (System.nanoTime() - startTime) / 1_000_000.0); - } else if (_ctx instanceof IdentifierExpressionContext) { - final long startTime = System.nanoTime(); - String propertyKeyPrefix = node.getText().substring(0, caretPosition - start); - propertyKeyPrefix = propertyKeyPrefix.replace(Proposer.PREFIX_MARKER, ""); - - final StringBuilder newQueryPattern = new StringBuilder(query); - newQueryPattern.replace(start, end + 1, "%s"); - - addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString()); - - METRICS_LOGGER.debug("proposals for property key {} took {} ms", propertyKeyPrefix, - (System.nanoTime() - startTime) / 1_000_000.0); - } - } - } - - @Override - public void syntaxError(final Recognizer recognizer, final Object offendingSymbol, final int line, - final int charPositionInLine, final String msg, final RecognitionException e) { - if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) { - - final CommonToken token = (CommonToken) offendingSymbol; - final String text = token.getText(); - - if ("and".startsWith(text)) { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and "); - - proposals.add(new Proposal(" and ", newQuery.toString(), true, newQuery.toString(), - charPositionInLine + text.length())); - } - if ("or".startsWith(text)) { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or "); - - proposals.add(new Proposal(" or ", newQuery.toString(), true, newQuery.toString(), - charPositionInLine + text.length())); - } - } - } - - private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) { - - final List availableKeys = dataStore.getAvailableFields(); - final List matchingKeys = CollectionUtils.filter(availableKeys, - s -> s.startsWith(propertyKeyPrefix)); - - matchingKeys.stream()// - .map(key -> { - - final String newQueryWithMarker = String.format(newQueryPattern, key + "=" + CARET_MARKER) - .replaceAll("\\s+", " "); - final int newCaretPosition = newQueryWithMarker.indexOf(CARET_MARKER); - final String newQuery = newQueryWithMarker.replace(CARET_MARKER, ""); - - return new Proposal(key, String.format(newQueryPattern, key + "=* "), true, newQuery, - newCaretPosition); - }).forEach(proposals::add); - } - - private boolean isEOF(final Object offendingSymbol) { - - if (offendingSymbol instanceof CommonToken) { - return ((CommonToken) offendingSymbol).getType() < 0; - } - - return false; - } - - @Override - public void visitErrorNode(final ErrorNode node) { - } - - @Override - public void enterEveryRule(final ParserRuleContext ctx) { - } - - @Override - public void exitEveryRule(final ParserRuleContext ctx) { - } - - @Override - public void enterStart(final StartContext ctx) { - } - - @Override - public void exitStart(final StartContext ctx) { - } - - @Override - public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) { - } - - @Override - public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) { - } - - @Override - public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) { - } - - @Override - public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) { - } - - @Override - public void enterNotExpression(final NotExpressionContext ctx) { - } - - @Override - public void exitNotExpression(final NotExpressionContext ctx) { - } - - @Override - public void enterParenExpression(final ParenExpressionContext ctx) { - } - - @Override - public void exitParenExpression(final ParenExpressionContext ctx) { - } - - @Override - public void enterPropertyExpression(final PropertyExpressionContext ctx) { - } - - @Override - public void exitPropertyExpression(final PropertyExpressionContext ctx) { - } - - @Override - public void enterIdentifierExpression(final IdentifierExpressionContext ctx) { - } - - @Override - public void exitIdentifierExpression(final IdentifierExpressionContext ctx) { - } - - @Override - public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { - } - - @Override - public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { - } - - private SortedSet getPropertyValuesByPrefix(final String propertyKey, - final String propertyValuePrefix) { - final SortedSet availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey); - - final SortedSet result = filterValues(availableValuesForKey, propertyValuePrefix); - - return result; - } - - @Override - public void enterEqual(final EqualContext ctx) { - } - - @Override - public void exitEqual(final EqualContext ctx) { - } - - private boolean isEOF(final TerminalNode node) { - return node.getSymbol().getType() < 0; - } - - private boolean containsCaret(final TerminalNode node) { - final int start = node.getSymbol().getStartIndex(); - final int end = node.getSymbol().getStopIndex(); - return start <= caretPosition && end + 1 >= caretPosition; - } - - @Override - public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex, - final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) { - } - - @Override - public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex, - final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) { - } - - @Override - public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex, - final int stopIndex, final int prediction, final ATNConfigSet configs) { - } - - @Override - public void enterListOfPropValues(final ListOfPropValuesContext ctx) { - } - - @Override - public void exitListOfPropValues(final ListOfPropValuesContext ctx) { - } - - @Override - public void enterEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) { - } - - @Override - public void exitEnclosedListOfPropValues(final EnclosedListOfPropValuesContext ctx) { - } - } - - public QueryCompletionPdbLangParser(final TokenStream input) { - super(input); - } - - static SortedSet filterValues(final Collection availableValues, final String valuePattern) { - final SortedSet result = new TreeSet<>(); - - final Pattern pattern = GloblikePattern.globlikeToRegex(valuePattern); - - for (final String value : availableValues) { - final Matcher matcher = pattern.matcher(value); - if (matcher.find() && !value.equals(valuePattern)) { - result.add(value); - } - } - - return result; - } - -} diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParserTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParserTest.java deleted file mode 100644 index 3d39023..0000000 --- a/data-store/src/test/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParserTest.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.lucares.pdb.datastore.lang; - -import java.util.ArrayList; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.testng.Assert; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -@Test -public class QueryCompletionPdbLangParserTest { - @DataProvider - public Object[][] providerPatterns() { - - final List result = new ArrayList<>(); - - // opinion-size-age-shape-colour-origin-material-purpose Noun - final List availableValues = new ArrayList<>(); - availableValues.add("Tim"); - availableValues.add("Timothy"); - availableValues.add("Tanja"); - availableValues.add("Danja"); - availableValues.add("Wanja"); - availableValues.add("BigOldGrey.Jennifer"); - availableValues.add("BigYoungGreen.Jennifer"); - availableValues.add("BigYoungBlue.Jenny"); - availableValues.add("SmallRoundBlue.Peter"); - - { - // infix does not match - final SortedSet expected = new TreeSet<>(); - result.add(new Object[] { availableValues, "nj", expected }); - } - { - final SortedSet expected = new TreeSet<>(); - expected.add("Danja"); - expected.add("Tanja"); - expected.add("Wanja"); - - result.add(new Object[] { availableValues, "*nj", expected }); - } - { - final SortedSet expected = new TreeSet<>(); - expected.add("BigYoungBlue.Jenny"); - - result.add(new Object[] { availableValues, "BYB", expected }); - } - { - final SortedSet expected = new TreeSet<>(); - expected.add("BigOldGrey.Jennifer"); - expected.add("BigYoungGreen.Jennifer"); - - result.add(new Object[] { availableValues, "B*Gr", expected }); - } - { - final SortedSet expected = new TreeSet<>(); - expected.add("BigOldGrey.Jennifer"); - expected.add("BigYoungGreen.Jennifer"); - expected.add("BigYoungBlue.Jenny"); - - result.add(new Object[] { availableValues, ".Jen", expected }); - } - { - final SortedSet expected = new TreeSet<>(); - expected.add("BigYoungBlue.Jenny"); - expected.add("BigYoungGreen.Jennifer"); - - result.add(new Object[] { availableValues, "BY.Jen", expected }); - } - - return result.toArray(new Object[0][]); - } - - @Test(dataProvider = "providerPatterns") - public void testPatterns(final List availableValues, final String valuePattern, - final SortedSet expectedValues) { - - final SortedSet actual = QueryCompletionPdbLangParser.filterValues(availableValues, valuePattern); - Assert.assertEquals(actual, expectedValues); - } -}