From a6a2236d185e220896f1e763c7c48d66761300d2 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sat, 18 Nov 2017 13:03:45 +0100 Subject: [PATCH] do not compute counts when proposing all keys --- .../org/lucares/pdb/datastore/Proposal.java | 44 +- .../pdb/datastore/internal/DataStore.java | 5 + .../pdb/datastore/internal/Proposer.java | 12 +- .../lang/QueryCompletionPdbLangParser.java | 552 +++++++++--------- .../pdb/datastore/internal/ProposerTest.java | 238 ++++---- .../java/org/lucares/pdbui/PdbController.java | 312 +++++----- pdb-ui/src/main/resources/log4j2.xml | 1 + 7 files changed, 584 insertions(+), 580 deletions(-) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java b/data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java index 4e45207..7980d09 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java @@ -5,19 +5,19 @@ public class Proposal implements Comparable { private final String proposedQuery; - private final long results; + private final boolean hasResults; - public Proposal(final String proposedTag, final String proposedQuery, final long results) { + public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults) { super(); this.proposedTag = proposedTag; this.proposedQuery = proposedQuery; - this.results = results; + this.hasResults = hasResults; } - public Proposal(final Proposal proposal, final long results) { + public Proposal(final Proposal proposal, final boolean hasResults) { this.proposedTag = proposal.proposedTag; this.proposedQuery = proposal.proposedQuery; - this.results = results; + this.hasResults = hasResults; } public String getProposedTag() { @@ -28,35 +28,42 @@ public class Proposal implements Comparable { return proposedQuery; } - public long getResults() { - return results; + public boolean hasResults() { + return hasResults; } @Override public String toString() { - return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", results=" + results + return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", hasResults=" + hasResults + "]"; } + + + @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode()); - result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode()); - result = prime * result + (int) (results ^ (results >>> 32)); + result = prime * result + (hasResults ? 1231 : 1237); + result = prime * result + + ((proposedQuery == null) ? 0 : proposedQuery.hashCode()); + result = prime * result + + ((proposedTag == null) ? 0 : proposedTag.hashCode()); return result; } @Override - public boolean equals(final Object obj) { + public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; - final Proposal other = (Proposal) obj; + Proposal other = (Proposal) obj; + if (hasResults != other.hasResults) + return false; if (proposedQuery == null) { if (other.proposedQuery != null) return false; @@ -67,18 +74,11 @@ public class Proposal implements Comparable { return false; } else if (!proposedTag.equals(other.proposedTag)) return false; - if (results != other.results) - return false; return true; } @Override - public int compareTo(final Proposal o) { - - if (results != o.results) { - return results < o.results ? 1 : -1; - } - - return proposedTag.compareToIgnoreCase(o.proposedTag); + public int compareTo(Proposal o) { + return proposedTag.compareTo(o.getProposedTag()); } } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java index 5b02124..a4facf4 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java @@ -22,8 +22,11 @@ import org.lucares.pdb.datastore.lang.Expression; import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor; import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds; import org.lucares.pdb.datastore.lang.QueryLanguageParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataStore { + private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.dataStore.executeQuery"); private static final String SUBDIR_STORAGE = "storage"; private static final String PDB_EXTENSION = ".pdb"; @@ -196,10 +199,12 @@ public class DataStore { } private IntList executeQuery(final String query) { + final long start = System.nanoTime(); final Expression expression = QueryLanguageParser.parse(query); final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(keyToValueToDocId, new AllDocIds(docIdToDoc)); final IntList docIdsList = expression.visit(visitor); + EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results " , query, (System.nanoTime() - start) / 1_000_000.0, docIdsList.size()); return docIdsList; } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java index 8e89bf2..349d91d 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java @@ -28,7 +28,7 @@ public class Proposer { result = ProposerParser.parse(query, dataStore, caretIndex); } - return CollectionUtils.filter(result, p -> p.getResults() >= 0); + return CollectionUtils.filter(result, p -> p.hasResults() ); } private SortedSet proposeForAllKeys() { @@ -37,23 +37,21 @@ public class Proposer { final Map fieldToQuery = CollectionUtils.createMapFromKeys(fields, f -> f + "=*"); - result = computeProposalsForQueries(fieldToQuery); + result = toProposalsForQueries(fieldToQuery); return result; } - - private SortedSet computeProposalsForQueries(final Map keyToQuery) { + + private SortedSet toProposalsForQueries(final Map keyToQuery) { final SortedSet result = new TreeSet<>(); for (final Entry e : keyToQuery.entrySet()) { final String key = e.getKey(); final String query = e.getValue(); - final int count = dataStore.count(query); - final Proposal proposal = new Proposal(key, query, count); + final Proposal proposal = new Proposal(key, query, true); result.add(proposal); } return result; } - } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java index 04dac91..cae815e 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java @@ -1,276 +1,276 @@ -package org.lucares.pdb.datastore.lang; - -import java.util.BitSet; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.CommonToken; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.tree.ErrorNode; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.lucares.pdb.datastore.Proposal; -import org.lucares.pdb.datastore.internal.DataStore; -import org.lucares.utils.CollectionUtils; - -public class QueryCompletionPdbLangParser extends PdbLangParser { - - public class Listener implements PdbLangListener, ANTLRErrorListener { - - private final int caretPosition; - private final DataStore dataStore; - private final SortedSet proposals = new TreeSet<>(); - private final String query; - - public Listener(final String query, final DataStore dataStore, final int caretPosition) { - this.query = query; - this.dataStore = dataStore; - this.caretPosition = caretPosition; - } - - public SortedSet getProposals() { - return proposals; - } - - @Override - public void visitTerminal(final TerminalNode node) { - if (containsCaret(node) && !isEOF(node)) { - final int start = node.getSymbol().getStartIndex(); - final int end = node.getSymbol().getStopIndex(); - - if (_ctx instanceof PropertyTerminalExpressionContext) { - final String propertyKey = _ctx.getParent().children.get(0).getText(); - final String propertyValuePrefix = node.getText().substring(0, caretPosition - start); - final SortedSet proposedValues = getPropertyValuesByPrefix(propertyKey, - propertyValuePrefix); - - proposedValues.stream()// - .map(v -> { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(start, end + 1, v + " "); - - return new Proposal(v, newQuery.toString(), -1); - }).map(p -> { - - final int count = dataStore.count(p.getProposedQuery()); - return new Proposal(p, count); - }).forEach(proposals::add); - - } else if (_ctx instanceof IdentifierExpressionContext) { - final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start); - - final StringBuilder newQueryPattern = new StringBuilder(query); - newQueryPattern.replace(start, end + 1, "%s"); - - addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString()); - } - } - } - - @Override - public void syntaxError(final Recognizer recognizer, final Object offendingSymbol, final int line, - final int charPositionInLine, final String msg, final RecognitionException e) { - if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) { - - final CommonToken token = (CommonToken) offendingSymbol; - final String text = token.getText(); - - if ("and".startsWith(text)) { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and "); - - proposals.add(new Proposal(" and ", newQuery.toString(), 1)); - } - if ("or".startsWith(text)) { - final StringBuilder newQuery = new StringBuilder(query); - newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or "); - - proposals.add(new Proposal(" or ", newQuery.toString(), 1)); - } - } - } - - private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) { - - final List availableKeys = dataStore.getAvailableFields(); - final List matchingKeys = CollectionUtils.filter(availableKeys, - s -> s.startsWith(propertyKeyPrefix)); - - matchingKeys.stream()// - .map(key -> { - - return new Proposal(key, String.format(newQueryPattern, key + "=* "), -1); - }).map(p -> { - - final String proposedQuery = p.getProposedQuery(); - final int count = count(proposedQuery); - return new Proposal(p, count); - }).forEach(proposals::add); - } - - private int count(final String proposedQuery) { - - try { - return dataStore.count(proposedQuery); - } catch (final SyntaxException e) { - return -1; - } - } - - private boolean isEOF(final Object offendingSymbol) { - - if (offendingSymbol instanceof CommonToken) { - return ((CommonToken) offendingSymbol).getType() < 0; - } - - return false; - } - - @Override - public void visitErrorNode(final ErrorNode node) { - } - - @Override - public void enterEveryRule(final ParserRuleContext ctx) { - } - - @Override - public void exitEveryRule(final ParserRuleContext ctx) { - } - - @Override - public void enterStart(final StartContext ctx) { - } - - @Override - public void exitStart(final StartContext ctx) { - } - - @Override - public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) { - } - - @Override - public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) { - } - - @Override - public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) { - } - - @Override - public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) { - } - - @Override - public void enterNotExpression(final NotExpressionContext ctx) { - } - - @Override - public void exitNotExpression(final NotExpressionContext ctx) { - } - - @Override - public void enterParenExpression(final ParenExpressionContext ctx) { - } - - @Override - public void exitParenExpression(final ParenExpressionContext ctx) { - } - - @Override - public void enterPropertyExpression(final PropertyExpressionContext ctx) { - } - - @Override - public void exitPropertyExpression(final PropertyExpressionContext ctx) { - } - - @Override - public void enterIdentifierExpression(final IdentifierExpressionContext ctx) { - } - - @Override - public void exitIdentifierExpression(final IdentifierExpressionContext ctx) { - } - - @Override - public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { - } - - @Override - public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { - // if (containsCaret(ctx)) { - // final int start = ctx.getStart().getStartIndex(); - // final int end = ctx.getStop().getStopIndex(); - // final int ruleIndex = _ctx.getRuleIndex(); - // - // final String prefix = ctx.getText().substring(0, caretPosition - - // start); - // ctx.getParent().children.get(0).getText(); - // - // proposals.addAll(getPropertyValuesByPrefix(prefix)); - // } - } - - private SortedSet getPropertyValuesByPrefix(final String propertyKey, - final String propertyValuePrefix) { - final SortedSet availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey); - - final SortedSet result = new TreeSet<>(); - - for (final String value : availableValuesForKey) { - if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) { - result.add(value); - } - } - - return result; - } - - @Override - public void enterEqual(final EqualContext ctx) { - } - - @Override - public void exitEqual(final EqualContext ctx) { - } - - private boolean isEOF(final TerminalNode node) { - return node.getSymbol().getType() < 0; - } - - private boolean containsCaret(final TerminalNode node) { - final int start = node.getSymbol().getStartIndex(); - final int end = node.getSymbol().getStopIndex(); - return start <= caretPosition && end + 1 >= caretPosition; - } - - @Override - public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex, - final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) { - } - - @Override - public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex, - final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) { - } - - @Override - public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex, - final int stopIndex, final int prediction, final ATNConfigSet configs) { - } - } - - public QueryCompletionPdbLangParser(final TokenStream input) { - super(input); - } - -} +package org.lucares.pdb.datastore.lang; + +import java.util.BitSet; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.antlr.v4.runtime.ANTLRErrorListener; +import org.antlr.v4.runtime.CommonToken; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.atn.ATNConfigSet; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.lucares.pdb.datastore.Proposal; +import org.lucares.pdb.datastore.internal.DataStore; +import org.lucares.utils.CollectionUtils; + +public class QueryCompletionPdbLangParser extends PdbLangParser { + + public class Listener implements PdbLangListener, ANTLRErrorListener { + + private final int caretPosition; + private final DataStore dataStore; + private final SortedSet proposals = new TreeSet<>(); + private final String query; + + public Listener(final String query, final DataStore dataStore, final int caretPosition) { + this.query = query; + this.dataStore = dataStore; + this.caretPosition = caretPosition; + } + + public SortedSet getProposals() { + return proposals; + } + + @Override + public void visitTerminal(final TerminalNode node) { + if (containsCaret(node) && !isEOF(node)) { + final int start = node.getSymbol().getStartIndex(); + final int end = node.getSymbol().getStopIndex(); + + if (_ctx instanceof PropertyTerminalExpressionContext) { + final String propertyKey = _ctx.getParent().children.get(0).getText(); + final String propertyValuePrefix = node.getText().substring(0, caretPosition - start); + final SortedSet proposedValues = getPropertyValuesByPrefix(propertyKey, + propertyValuePrefix); + + proposedValues.stream()// + .map(v -> { + final StringBuilder newQuery = new StringBuilder(query); + newQuery.replace(start, end + 1, v + " "); // insert the terminal into the query + + return new Proposal(v, newQuery.toString(), false); + }).map(p -> { + + final int count = dataStore.count(p.getProposedQuery()); + return new Proposal(p, count> 0); + }).forEach(proposals::add); + + } else if (_ctx instanceof IdentifierExpressionContext) { + final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start); + + final StringBuilder newQueryPattern = new StringBuilder(query); + newQueryPattern.replace(start, end + 1, "%s"); + + addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString()); + } + } + } + + @Override + public void syntaxError(final Recognizer recognizer, final Object offendingSymbol, final int line, + final int charPositionInLine, final String msg, final RecognitionException e) { + if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) { + + final CommonToken token = (CommonToken) offendingSymbol; + final String text = token.getText(); + + if ("and".startsWith(text)) { + final StringBuilder newQuery = new StringBuilder(query); + newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and "); + + proposals.add(new Proposal(" and ", newQuery.toString(), true)); + } + if ("or".startsWith(text)) { + final StringBuilder newQuery = new StringBuilder(query); + newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or "); + + proposals.add(new Proposal(" or ", newQuery.toString(), true)); + } + } + } + + private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) { + + final List availableKeys = dataStore.getAvailableFields(); + final List matchingKeys = CollectionUtils.filter(availableKeys, + s -> s.startsWith(propertyKeyPrefix)); + + matchingKeys.stream()// + .map(key -> { + + return new Proposal(key, String.format(newQueryPattern, key + "=* "), false); + }).map(p -> { + + final String proposedQuery = p.getProposedQuery(); + final int count = count(proposedQuery); + return new Proposal(p, count > 0); + }).forEach(proposals::add); + } + + private int count(final String proposedQuery) { + + try { + return dataStore.count(proposedQuery); + } catch (final SyntaxException e) { + return -1; + } + } + + private boolean isEOF(final Object offendingSymbol) { + + if (offendingSymbol instanceof CommonToken) { + return ((CommonToken) offendingSymbol).getType() < 0; + } + + return false; + } + + @Override + public void visitErrorNode(final ErrorNode node) { + } + + @Override + public void enterEveryRule(final ParserRuleContext ctx) { + } + + @Override + public void exitEveryRule(final ParserRuleContext ctx) { + } + + @Override + public void enterStart(final StartContext ctx) { + } + + @Override + public void exitStart(final StartContext ctx) { + } + + @Override + public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) { + } + + @Override + public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) { + } + + @Override + public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) { + } + + @Override + public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) { + } + + @Override + public void enterNotExpression(final NotExpressionContext ctx) { + } + + @Override + public void exitNotExpression(final NotExpressionContext ctx) { + } + + @Override + public void enterParenExpression(final ParenExpressionContext ctx) { + } + + @Override + public void exitParenExpression(final ParenExpressionContext ctx) { + } + + @Override + public void enterPropertyExpression(final PropertyExpressionContext ctx) { + } + + @Override + public void exitPropertyExpression(final PropertyExpressionContext ctx) { + } + + @Override + public void enterIdentifierExpression(final IdentifierExpressionContext ctx) { + } + + @Override + public void exitIdentifierExpression(final IdentifierExpressionContext ctx) { + } + + @Override + public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { + } + + @Override + public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { + // if (containsCaret(ctx)) { + // final int start = ctx.getStart().getStartIndex(); + // final int end = ctx.getStop().getStopIndex(); + // final int ruleIndex = _ctx.getRuleIndex(); + // + // final String prefix = ctx.getText().substring(0, caretPosition - + // start); + // ctx.getParent().children.get(0).getText(); + // + // proposals.addAll(getPropertyValuesByPrefix(prefix)); + // } + } + + private SortedSet getPropertyValuesByPrefix(final String propertyKey, + final String propertyValuePrefix) { + final SortedSet availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey); + + final SortedSet result = new TreeSet<>(); + + for (final String value : availableValuesForKey) { + if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) { + result.add(value); + } + } + + return result; + } + + @Override + public void enterEqual(final EqualContext ctx) { + } + + @Override + public void exitEqual(final EqualContext ctx) { + } + + private boolean isEOF(final TerminalNode node) { + return node.getSymbol().getType() < 0; + } + + private boolean containsCaret(final TerminalNode node) { + final int start = node.getSymbol().getStartIndex(); + final int end = node.getSymbol().getStopIndex(); + return start <= caretPosition && end + 1 >= caretPosition; + } + + @Override + public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex, + final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) { + } + + @Override + public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex, + final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) { + } + + @Override + public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex, + final int stopIndex, final int prediction, final ATNConfigSet configs) { + } + } + + public QueryCompletionPdbLangParser(final TokenStream input) { + super(input); + } + +} diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java index f4b84ee..530bbcf 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java @@ -1,119 +1,119 @@ -package org.lucares.pdb.datastore.internal; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.lucares.pdb.api.Tags; -import org.lucares.pdb.datastore.PdbDB; -import org.lucares.pdb.datastore.Proposal; -import org.lucares.utils.file.FileUtils; -import org.testng.Assert; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -@Test -public class ProposerTest { - - private Path dataDirectory; - private PdbDB db; - private Map tagsToPath; - - @BeforeClass - public void beforeClass() throws Exception { - dataDirectory = Files.createTempDirectory("pdb"); - initDatabase(); - } - - @AfterClass - public void afterClass() throws IOException { - FileUtils.delete(dataDirectory); - db = null; - tagsToPath = null; - } - - private void initDatabase() throws Exception { - tagsToPath = new LinkedHashMap<>(); - final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim"); - final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy"); - final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer"); - final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer"); - final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny"); - final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim"); - - tagsToPath.put(eagleTim, null); - tagsToPath.put(eagleTimothy, null); - tagsToPath.put(pigeonJennifer, null); - tagsToPath.put(flamingoJennifer, null); - tagsToPath.put(labradorJenny, null); - tagsToPath.put(labradorTim, null); - - db = new PdbDB(dataDirectory); - - for (final Tags tags : tagsToPath.keySet()) { - final Path newFile = db.createNewFile(tags); - tagsToPath.put(tags, newFile); - } - } - - public void testEmptyQuery() throws Exception { - - assertProposals("", 0, // - new Proposal("name", "name=*", 6), // - new Proposal("bird", "bird=*", 4), // - new Proposal("dog", "dog=*", 2)// - ); - - assertProposals(" ", 1, // - new Proposal("name", "name=*", 6), // - new Proposal("bird", "bird=*", 4), // - new Proposal("dog", "dog=*", 2)// - ); - } - - public void testPrefixOfKey() throws Exception { - assertProposals("bi", 2, // - new Proposal("bird", "bird=* ", 4) // - ); - assertProposals("bird", 4, // - new Proposal("bird", "bird=* ", 4) // - ); - } - - public void testPrefixOfValue() throws Exception { - assertProposals("name =Tim", 9, // - new Proposal("Timothy", "name =Timothy ", 1) - ); - - /* - assertProposals("name =Je", 8, // - new Proposal("Jennifer", "name =Jennifer ", 2), // - new Proposal("Jenny", "name =Jenny ", 1) // - ); - - - assertProposals("bird=eagle and n", 16, // - new Proposal("name", "bird=eagle and name=* ", 1) // - ); - */ - } - - private void assertProposals(final String query, final int caretIndex, final Proposal... expected) - throws InterruptedException { - - final List actual = db.propose(query, caretIndex); - final List expectedList = Arrays.asList(expected); - Collections.sort(expectedList); - - System.out.println("\n\n--- " + query + " ---"); - System.out.println("actual : " + actual); - System.out.println("expected: " + expectedList); - Assert.assertEquals(expectedList, actual); - } -} +package org.lucares.pdb.datastore.internal; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.lucares.pdb.api.Tags; +import org.lucares.pdb.datastore.PdbDB; +import org.lucares.pdb.datastore.Proposal; +import org.lucares.utils.file.FileUtils; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Test +public class ProposerTest { + + private Path dataDirectory; + private PdbDB db; + private Map tagsToPath; + + @BeforeClass + public void beforeClass() throws Exception { + dataDirectory = Files.createTempDirectory("pdb"); + initDatabase(); + } + + @AfterClass + public void afterClass() throws IOException { + FileUtils.delete(dataDirectory); + db = null; + tagsToPath = null; + } + + private void initDatabase() throws Exception { + tagsToPath = new LinkedHashMap<>(); + final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim"); + final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy"); + final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer"); + final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer"); + final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny"); + final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim"); + + tagsToPath.put(eagleTim, null); + tagsToPath.put(eagleTimothy, null); + tagsToPath.put(pigeonJennifer, null); + tagsToPath.put(flamingoJennifer, null); + tagsToPath.put(labradorJenny, null); + tagsToPath.put(labradorTim, null); + + db = new PdbDB(dataDirectory); + + for (final Tags tags : tagsToPath.keySet()) { + final Path newFile = db.createNewFile(tags); + tagsToPath.put(tags, newFile); + } + } + + public void testEmptyQuery() throws Exception { + + assertProposals("", 0, // + new Proposal("name", "name=*", true), // + new Proposal("bird", "bird=*", true), // + new Proposal("dog", "dog=*", true)// + ); + + assertProposals(" ", 1, // + new Proposal("name", "name=*", true), // + new Proposal("bird", "bird=*", true), // + new Proposal("dog", "dog=*", true)// + ); + } + + public void testPrefixOfKey() throws Exception { + assertProposals("bi", 2, // + new Proposal("bird", "bird=* ", true) // + ); + assertProposals("bird", 4, // + new Proposal("bird", "bird=* ", true) // + ); + } + + public void testPrefixOfValue() throws Exception { + assertProposals("name =Tim", 9, // + new Proposal("Timothy", "name =Timothy ", true) + ); + + assertProposals("name =Je", 8, // + new Proposal("Jennifer", "name =Jennifer ", true), // + new Proposal("Jenny", "name =Jenny ", true) // + ); + + + assertProposals("bird=eagle and n", 16, // + new Proposal("name", "bird=eagle and name=* ", true) // + ); + /* + */ + } + + private void assertProposals(final String query, final int caretIndex, final Proposal... expected) + throws InterruptedException { + + final List actual = db.propose(query, caretIndex); + final List expectedList = Arrays.asList(expected); + Collections.sort(expectedList); + + System.out.println("\n\n--- " + query + " ---"); + System.out.println("actual : " + actual); + System.out.println("expected: " + expectedList); + Assert.assertEquals(expectedList, actual); + } +} diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 13f07e8..52101a7 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -1,156 +1,156 @@ -package org.lucares.pdbui; - -import java.text.Collator; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.SortedSet; - -import org.lucares.pdb.datastore.Proposal; -import org.lucares.pdb.plot.api.PlotSettings; -import org.lucares.pdbui.domain.AutocompleteProposal; -import org.lucares.pdbui.domain.AutocompleteProposalByValue; -import org.lucares.pdbui.domain.AutocompleteResponse; -import org.lucares.pdbui.domain.PlotRequest; -import org.lucares.pdbui.domain.PlotResponse; -import org.lucares.performance.db.PerformanceDb; -import org.lucares.recommind.logs.DataSeries; -import org.lucares.recommind.logs.InternalPlottingException; -import org.lucares.recommind.logs.NoDataPointsException; -import org.lucares.recommind.logs.PlotResult; -import org.lucares.recommind.logs.Plotter; -import org.lucares.utils.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.servlet.ModelAndView; - -@Controller -@EnableAutoConfiguration -public class PdbController implements HardcodedValues { - - private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class); - - private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00"); - private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59"); - - private final Plotter plotter; - private final PerformanceDb db; - - public PdbController(final PerformanceDb db, final Plotter plotter) { - this.db = db; - this.plotter = plotter; - } - - @GetMapping("/") - public ModelAndView index() { - final String view = "main"; - final Map model = new HashMap<>(); - model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN)); - model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END)); - return new ModelAndView(view, model); - } - - @RequestMapping(path = "/plots", // - method = RequestMethod.POST, // - consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // - produces = MediaType.APPLICATION_JSON_UTF8_VALUE // - ) - @ResponseBody - PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException { - - final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request); - - try { - final PlotResult result = plotter.plot(plotSettings); - - final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName(); - LOGGER.trace("image url: {}", imageUrl); - System.gc(); - - return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl); - } catch (final NoDataPointsException e) { - throw new NotFoundException(e); - } - } - - @RequestMapping(path = "/autocomplete", // - method = RequestMethod.GET, // - consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, // - produces = MediaType.APPLICATION_JSON_UTF8_VALUE // - ) - @ResponseBody - AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query, - @RequestParam(name = "caretIndex") final int caretIndex) { - - final AutocompleteResponse result = new AutocompleteResponse(); - final int zeroBasedCaretIndex = caretIndex - 1; - - final List proposals = db.autocomplete(query, zeroBasedCaretIndex); - final List nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.getResults() > 0); - - final List autocompleteProposals = toAutocompleteProposals(nonEmptyProposals); - Collections.sort(autocompleteProposals, new AutocompleteProposalByValue()); - - result.setProposals(autocompleteProposals); - return result; - } - - @RequestMapping(path = "/fields", // - method = RequestMethod.GET, // - consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // - produces = MediaType.APPLICATION_JSON_UTF8_VALUE // - ) - @ResponseBody - List fields() { - final List fields = db.getFields(); - - fields.sort(Collator.getInstance(Locale.ENGLISH)); - - return fields; - } - - @RequestMapping(path = "/fields/{fieldName}/values", // - method = RequestMethod.GET, // - consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // - produces = MediaType.APPLICATION_JSON_UTF8_VALUE // - ) - @ResponseBody - SortedSet fields(@PathVariable(name = "fieldName") final String fieldName, - @RequestParam(name = "query") final String query) { - - final SortedSet fields = db.getFieldsValues(query, fieldName); - - return fields; - } - - private List toAutocompleteProposals(final List proposals) { - - final List result = new ArrayList<>(); - - for (final Proposal proposal : proposals) { - final AutocompleteProposal e = new AutocompleteProposal(); - e.setValue(proposal.getProposedTag()); - e.setProposedQuery(proposal.getProposedQuery()); - - result.add(e); - } - - return result; - } - -} +package org.lucares.pdbui; + +import java.text.Collator; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedSet; + +import org.lucares.pdb.datastore.Proposal; +import org.lucares.pdb.plot.api.PlotSettings; +import org.lucares.pdbui.domain.AutocompleteProposal; +import org.lucares.pdbui.domain.AutocompleteProposalByValue; +import org.lucares.pdbui.domain.AutocompleteResponse; +import org.lucares.pdbui.domain.PlotRequest; +import org.lucares.pdbui.domain.PlotResponse; +import org.lucares.performance.db.PerformanceDb; +import org.lucares.recommind.logs.DataSeries; +import org.lucares.recommind.logs.InternalPlottingException; +import org.lucares.recommind.logs.NoDataPointsException; +import org.lucares.recommind.logs.PlotResult; +import org.lucares.recommind.logs.Plotter; +import org.lucares.utils.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.servlet.ModelAndView; + +@Controller +@EnableAutoConfiguration +public class PdbController implements HardcodedValues { + + private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class); + + private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00"); + private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59"); + + private final Plotter plotter; + private final PerformanceDb db; + + public PdbController(final PerformanceDb db, final Plotter plotter) { + this.db = db; + this.plotter = plotter; + } + + @GetMapping("/") + public ModelAndView index() { + final String view = "main"; + final Map model = new HashMap<>(); + model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN)); + model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END)); + return new ModelAndView(view, model); + } + + @RequestMapping(path = "/plots", // + method = RequestMethod.POST, // + consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // + produces = MediaType.APPLICATION_JSON_UTF8_VALUE // + ) + @ResponseBody + PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException { + + final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request); + + try { + final PlotResult result = plotter.plot(plotSettings); + + final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName(); + LOGGER.trace("image url: {}", imageUrl); + System.gc(); + + return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl); + } catch (final NoDataPointsException e) { + throw new NotFoundException(e); + } + } + + @RequestMapping(path = "/autocomplete", // + method = RequestMethod.GET, // + consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, // + produces = MediaType.APPLICATION_JSON_UTF8_VALUE // + ) + @ResponseBody + AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query, + @RequestParam(name = "caretIndex") final int caretIndex) { + + final AutocompleteResponse result = new AutocompleteResponse(); + final int zeroBasedCaretIndex = caretIndex - 1; + + final List proposals = db.autocomplete(query, zeroBasedCaretIndex); + final List nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults() ); + + final List autocompleteProposals = toAutocompleteProposals(nonEmptyProposals); + Collections.sort(autocompleteProposals, new AutocompleteProposalByValue()); + + result.setProposals(autocompleteProposals); + return result; + } + + @RequestMapping(path = "/fields", // + method = RequestMethod.GET, // + consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // + produces = MediaType.APPLICATION_JSON_UTF8_VALUE // + ) + @ResponseBody + List fields() { + final List fields = db.getFields(); + + fields.sort(Collator.getInstance(Locale.ENGLISH)); + + return fields; + } + + @RequestMapping(path = "/fields/{fieldName}/values", // + method = RequestMethod.GET, // + consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // + produces = MediaType.APPLICATION_JSON_UTF8_VALUE // + ) + @ResponseBody + SortedSet fields(@PathVariable(name = "fieldName") final String fieldName, + @RequestParam(name = "query") final String query) { + + final SortedSet fields = db.getFieldsValues(query, fieldName); + + return fields; + } + + private List toAutocompleteProposals(final List proposals) { + + final List result = new ArrayList<>(); + + for (final Proposal proposal : proposals) { + final AutocompleteProposal e = new AutocompleteProposal(); + e.setValue(proposal.getProposedTag()); + e.setProposedQuery(proposal.getProposedQuery()); + + result.add(e); + } + + return result; + } + +} diff --git a/pdb-ui/src/main/resources/log4j2.xml b/pdb-ui/src/main/resources/log4j2.xml index e5bb6c7..5de201a 100644 --- a/pdb-ui/src/main/resources/log4j2.xml +++ b/pdb-ui/src/main/resources/log4j2.xml @@ -21,5 +21,6 @@ + \ No newline at end of file