andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

do not compute counts when proposing all keys

Browse Source
This commit is contained in:
Andreas Huber
2017-11-18 13:03:45 +01:00
parent 14d1367c4a
commit a6a2236d18
7 changed files with 584 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -5,19 +5,19 @@ public class Proposal implements Comparable<Proposal> {
private final String proposedQuery; private final String proposedQuery;
private final long results; private final boolean hasResults;
public Proposal(final String proposedTag, final String proposedQuery, final long results) { public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults) {
super(); super();
this.proposedTag = proposedTag; this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery; this.proposedQuery = proposedQuery;
this.results = results; this.hasResults = hasResults;
} }
public Proposal(final Proposal proposal, final long results) { public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag; this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery; this.proposedQuery = proposal.proposedQuery;
this.results = results; this.hasResults = hasResults;
} }
public String getProposedTag() { public String getProposedTag() {
@@ -28,35 +28,42 @@ public class Proposal implements Comparable<Proposal> {
return proposedQuery; return proposedQuery;
} }
public long getResults() { public boolean hasResults() {
return results; return hasResults;
} }
@Override @Override
public String toString() { public String toString() {
return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", results=" + results return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", hasResults=" + hasResults
+ "]"; + "]";
} }
@Override @Override
public int hashCode() { public int hashCode() {
final int prime = 31; final int prime = 31;
int result = 1; int result = 1;
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode()); result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode()); result = prime * result
result = prime * result + (int) (results ^ (results >>> 32)); + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result
+ ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result; return result;
} }
@Override @Override
public boolean equals(final Object obj) { public boolean equals(Object obj) {
if (this == obj) if (this == obj)
return true; return true;
if (obj == null) if (obj == null)
return false; return false;
if (getClass() != obj.getClass()) if (getClass() != obj.getClass())
return false; return false;
final Proposal other = (Proposal) obj; Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (proposedQuery == null) { if (proposedQuery == null) {
if (other.proposedQuery != null) if (other.proposedQuery != null)
return false; return false;
@@ -67,18 +74,11 @@ public class Proposal implements Comparable<Proposal> {
return false; return false;
} else if (!proposedTag.equals(other.proposedTag)) } else if (!proposedTag.equals(other.proposedTag))
return false; return false;
if (results != other.results)
return false;
return true; return true;
} }
@Override @Override
public int compareTo(final Proposal o) { public int compareTo(Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
if (results != o.results) {
return results < o.results ? 1 : -1;
}
return proposedTag.compareToIgnoreCase(o.proposedTag);
} }
} }

5
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -22,8 +22,11 @@ import org.lucares.pdb.datastore.lang.Expression;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor; import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds; import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds;
import org.lucares.pdb.datastore.lang.QueryLanguageParser; import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore { public class DataStore {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final String SUBDIR_STORAGE = "storage"; private static final String SUBDIR_STORAGE = "storage";
private static final String PDB_EXTENSION = ".pdb"; private static final String PDB_EXTENSION = ".pdb";
@@ -196,10 +199,12 @@ public class DataStore {
} }
private IntList executeQuery(final String query) { private IntList executeQuery(final String query) {
final long start = System.nanoTime();
final Expression expression = QueryLanguageParser.parse(query); final Expression expression = QueryLanguageParser.parse(query);
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(keyToValueToDocId, final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(keyToValueToDocId,
new AllDocIds(docIdToDoc)); new AllDocIds(docIdToDoc));
final IntList docIdsList = expression.visit(visitor); final IntList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results " , query, (System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList; return docIdsList;
} }

12
data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java
View File

@@ -28,7 +28,7 @@ public class Proposer {
result = ProposerParser.parse(query, dataStore, caretIndex); result = ProposerParser.parse(query, dataStore, caretIndex);
} }
return CollectionUtils.filter(result, p -> p.getResults() >= 0); return CollectionUtils.filter(result, p -> p.hasResults() );
} }
private SortedSet<Proposal> proposeForAllKeys() { private SortedSet<Proposal> proposeForAllKeys() {
@@ -37,23 +37,21 @@ public class Proposer {
final Map<String, String> fieldToQuery = CollectionUtils.createMapFromKeys(fields, f -> f + "=*"); final Map<String, String> fieldToQuery = CollectionUtils.createMapFromKeys(fields, f -> f + "=*");
result = computeProposalsForQueries(fieldToQuery); result = toProposalsForQueries(fieldToQuery);
return result; return result;
} }
private SortedSet<Proposal> computeProposalsForQueries(final Map<String, String> keyToQuery) { private SortedSet<Proposal> toProposalsForQueries(final Map<String, String> keyToQuery) {
final SortedSet<Proposal> result = new TreeSet<>(); final SortedSet<Proposal> result = new TreeSet<>();
for (final Entry<String, String> e : keyToQuery.entrySet()) { for (final Entry<String, String> e : keyToQuery.entrySet()) {
final String key = e.getKey(); final String key = e.getKey();
final String query = e.getValue(); final String query = e.getValue();
final int count = dataStore.count(query);
final Proposal proposal = new Proposal(key, query, count); final Proposal proposal = new Proposal(key, query, true);
result.add(proposal); result.add(proposal);
} }
return result; return result;
} }
} }

552
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java
View File

@@ -1,276 +1,276 @@
package org.lucares.pdb.datastore.lang; package org.lucares.pdb.datastore.lang;
import java.util.BitSet; import java.util.BitSet;
import java.util.List; import java.util.List;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeSet; import java.util.TreeSet;
import org.antlr.v4.runtime.ANTLRErrorListener; import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.CommonToken; import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Parser; import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATNConfigSet; import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode; import org.antlr.v4.runtime.tree.TerminalNode;
import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.DataStore; import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.CollectionUtils; import org.lucares.utils.CollectionUtils;
public class QueryCompletionPdbLangParser extends PdbLangParser { public class QueryCompletionPdbLangParser extends PdbLangParser {
public class Listener implements PdbLangListener, ANTLRErrorListener { public class Listener implements PdbLangListener, ANTLRErrorListener {
private final int caretPosition; private final int caretPosition;
private final DataStore dataStore; private final DataStore dataStore;
private final SortedSet<Proposal> proposals = new TreeSet<>(); private final SortedSet<Proposal> proposals = new TreeSet<>();
private final String query; private final String query;
public Listener(final String query, final DataStore dataStore, final int caretPosition) { public Listener(final String query, final DataStore dataStore, final int caretPosition) {
this.query = query; this.query = query;
this.dataStore = dataStore; this.dataStore = dataStore;
this.caretPosition = caretPosition; this.caretPosition = caretPosition;
} }
public SortedSet<Proposal> getProposals() { public SortedSet<Proposal> getProposals() {
return proposals; return proposals;
} }
@Override @Override
public void visitTerminal(final TerminalNode node) { public void visitTerminal(final TerminalNode node) {
if (containsCaret(node) && !isEOF(node)) { if (containsCaret(node) && !isEOF(node)) {
final int start = node.getSymbol().getStartIndex(); final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex(); final int end = node.getSymbol().getStopIndex();
if (_ctx instanceof PropertyTerminalExpressionContext) { if (_ctx instanceof PropertyTerminalExpressionContext) {
final String propertyKey = _ctx.getParent().children.get(0).getText(); final String propertyKey = _ctx.getParent().children.get(0).getText();
final String propertyValuePrefix = node.getText().substring(0, caretPosition - start); final String propertyValuePrefix = node.getText().substring(0, caretPosition - start);
final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey, final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey,
propertyValuePrefix); propertyValuePrefix);
proposedValues.stream()// proposedValues.stream()//
.map(v -> { .map(v -> {
final StringBuilder newQuery = new StringBuilder(query); final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(start, end + 1, v + " "); newQuery.replace(start, end + 1, v + " "); // insert the terminal into the query
return new Proposal(v, newQuery.toString(), -1); return new Proposal(v, newQuery.toString(), false);
}).map(p -> { }).map(p -> {
final int count = dataStore.count(p.getProposedQuery()); final int count = dataStore.count(p.getProposedQuery());
return new Proposal(p, count); return new Proposal(p, count> 0);
}).forEach(proposals::add); }).forEach(proposals::add);
} else if (_ctx instanceof IdentifierExpressionContext) { } else if (_ctx instanceof IdentifierExpressionContext) {
final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start); final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start);
final StringBuilder newQueryPattern = new StringBuilder(query); final StringBuilder newQueryPattern = new StringBuilder(query);
newQueryPattern.replace(start, end + 1, "%s"); newQueryPattern.replace(start, end + 1, "%s");
addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString()); addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString());
} }
} }
} }
@Override @Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) { final int charPositionInLine, final String msg, final RecognitionException e) {
if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) { if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) {
final CommonToken token = (CommonToken) offendingSymbol; final CommonToken token = (CommonToken) offendingSymbol;
final String text = token.getText(); final String text = token.getText();
if ("and".startsWith(text)) { if ("and".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query); final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and "); newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and ");
proposals.add(new Proposal(" and ", newQuery.toString(), 1)); proposals.add(new Proposal(" and ", newQuery.toString(), true));
} }
if ("or".startsWith(text)) { if ("or".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query); final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or "); newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or ");
proposals.add(new Proposal(" or ", newQuery.toString(), 1)); proposals.add(new Proposal(" or ", newQuery.toString(), true));
} }
} }
} }
private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) { private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) {
final List<String> availableKeys = dataStore.getAvailableFields(); final List<String> availableKeys = dataStore.getAvailableFields();
final List<String> matchingKeys = CollectionUtils.filter(availableKeys, final List<String> matchingKeys = CollectionUtils.filter(availableKeys,
s -> s.startsWith(propertyKeyPrefix)); s -> s.startsWith(propertyKeyPrefix));
matchingKeys.stream()// matchingKeys.stream()//
.map(key -> { .map(key -> {
return new Proposal(key, String.format(newQueryPattern, key + "=* "), -1); return new Proposal(key, String.format(newQueryPattern, key + "=* "), false);
}).map(p -> { }).map(p -> {
final String proposedQuery = p.getProposedQuery(); final String proposedQuery = p.getProposedQuery();
final int count = count(proposedQuery); final int count = count(proposedQuery);
return new Proposal(p, count); return new Proposal(p, count > 0);
}).forEach(proposals::add); }).forEach(proposals::add);
} }
private int count(final String proposedQuery) { private int count(final String proposedQuery) {
try { try {
return dataStore.count(proposedQuery); return dataStore.count(proposedQuery);
} catch (final SyntaxException e) { } catch (final SyntaxException e) {
return -1; return -1;
} }
} }
private boolean isEOF(final Object offendingSymbol) { private boolean isEOF(final Object offendingSymbol) {
if (offendingSymbol instanceof CommonToken) { if (offendingSymbol instanceof CommonToken) {
return ((CommonToken) offendingSymbol).getType() < 0; return ((CommonToken) offendingSymbol).getType() < 0;
} }
return false; return false;
} }
@Override @Override
public void visitErrorNode(final ErrorNode node) { public void visitErrorNode(final ErrorNode node) {
} }
@Override @Override
public void enterEveryRule(final ParserRuleContext ctx) { public void enterEveryRule(final ParserRuleContext ctx) {
} }
@Override @Override
public void exitEveryRule(final ParserRuleContext ctx) { public void exitEveryRule(final ParserRuleContext ctx) {
} }
@Override @Override
public void enterStart(final StartContext ctx) { public void enterStart(final StartContext ctx) {
} }
@Override @Override
public void exitStart(final StartContext ctx) { public void exitStart(final StartContext ctx) {
} }
@Override @Override
public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) { public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) {
} }
@Override @Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) { public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
} }
@Override @Override
public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) { public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) {
} }
@Override @Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) { public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
} }
@Override @Override
public void enterNotExpression(final NotExpressionContext ctx) { public void enterNotExpression(final NotExpressionContext ctx) {
} }
@Override @Override
public void exitNotExpression(final NotExpressionContext ctx) { public void exitNotExpression(final NotExpressionContext ctx) {
} }
@Override @Override
public void enterParenExpression(final ParenExpressionContext ctx) { public void enterParenExpression(final ParenExpressionContext ctx) {
} }
@Override @Override
public void exitParenExpression(final ParenExpressionContext ctx) { public void exitParenExpression(final ParenExpressionContext ctx) {
} }
@Override @Override
public void enterPropertyExpression(final PropertyExpressionContext ctx) { public void enterPropertyExpression(final PropertyExpressionContext ctx) {
} }
@Override @Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) { public void exitPropertyExpression(final PropertyExpressionContext ctx) {
} }
@Override @Override
public void enterIdentifierExpression(final IdentifierExpressionContext ctx) { public void enterIdentifierExpression(final IdentifierExpressionContext ctx) {
} }
@Override @Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) { public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
} }
@Override @Override
public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
} }
@Override @Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) { public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
// if (containsCaret(ctx)) { // if (containsCaret(ctx)) {
// final int start = ctx.getStart().getStartIndex(); // final int start = ctx.getStart().getStartIndex();
// final int end = ctx.getStop().getStopIndex(); // final int end = ctx.getStop().getStopIndex();
// final int ruleIndex = _ctx.getRuleIndex(); // final int ruleIndex = _ctx.getRuleIndex();
// //
// final String prefix = ctx.getText().substring(0, caretPosition - // final String prefix = ctx.getText().substring(0, caretPosition -
// start); // start);
// ctx.getParent().children.get(0).getText(); // ctx.getParent().children.get(0).getText();
// //
// proposals.addAll(getPropertyValuesByPrefix(prefix)); // proposals.addAll(getPropertyValuesByPrefix(prefix));
// } // }
} }
private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey, private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey,
final String propertyValuePrefix) { final String propertyValuePrefix) {
final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey); final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey);
final SortedSet<String> result = new TreeSet<>(); final SortedSet<String> result = new TreeSet<>();
for (final String value : availableValuesForKey) { for (final String value : availableValuesForKey) {
if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) { if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) {
result.add(value); result.add(value);
} }
} }
return result; return result;
} }
@Override @Override
public void enterEqual(final EqualContext ctx) { public void enterEqual(final EqualContext ctx) {
} }
@Override @Override
public void exitEqual(final EqualContext ctx) { public void exitEqual(final EqualContext ctx) {
} }
private boolean isEOF(final TerminalNode node) { private boolean isEOF(final TerminalNode node) {
return node.getSymbol().getType() < 0; return node.getSymbol().getType() < 0;
} }
private boolean containsCaret(final TerminalNode node) { private boolean containsCaret(final TerminalNode node) {
final int start = node.getSymbol().getStartIndex(); final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex(); final int end = node.getSymbol().getStopIndex();
return start <= caretPosition && end + 1 >= caretPosition; return start <= caretPosition && end + 1 >= caretPosition;
} }
@Override @Override
public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex, public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex,
final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) { final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) {
} }
@Override @Override
public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex, public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) { final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) {
} }
@Override @Override
public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex, public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final int prediction, final ATNConfigSet configs) { final int stopIndex, final int prediction, final ATNConfigSet configs) {
} }
} }
public QueryCompletionPdbLangParser(final TokenStream input) { public QueryCompletionPdbLangParser(final TokenStream input) {
super(input); super(input);
} }
} }

238
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -1,119 +1,119 @@
package org.lucares.pdb.datastore.internal; package org.lucares.pdb.datastore.internal;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.lucares.pdb.api.Tags; import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbDB; import org.lucares.pdb.datastore.PdbDB;
import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.Proposal;
import org.lucares.utils.file.FileUtils; import org.lucares.utils.file.FileUtils;
import org.testng.Assert; import org.testng.Assert;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@Test @Test
public class ProposerTest { public class ProposerTest {
private Path dataDirectory; private Path dataDirectory;
private PdbDB db; private PdbDB db;
private Map<Tags, Path> tagsToPath; private Map<Tags, Path> tagsToPath;
@BeforeClass @BeforeClass
public void beforeClass() throws Exception { public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb"); dataDirectory = Files.createTempDirectory("pdb");
initDatabase(); initDatabase();
} }
@AfterClass @AfterClass
public void afterClass() throws IOException { public void afterClass() throws IOException {
FileUtils.delete(dataDirectory); FileUtils.delete(dataDirectory);
db = null; db = null;
tagsToPath = null; tagsToPath = null;
} }
private void initDatabase() throws Exception { private void initDatabase() throws Exception {
tagsToPath = new LinkedHashMap<>(); tagsToPath = new LinkedHashMap<>();
final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim"); final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy"); final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer"); final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer"); final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny"); final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim"); final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim");
tagsToPath.put(eagleTim, null); tagsToPath.put(eagleTim, null);
tagsToPath.put(eagleTimothy, null); tagsToPath.put(eagleTimothy, null);
tagsToPath.put(pigeonJennifer, null); tagsToPath.put(pigeonJennifer, null);
tagsToPath.put(flamingoJennifer, null); tagsToPath.put(flamingoJennifer, null);
tagsToPath.put(labradorJenny, null); tagsToPath.put(labradorJenny, null);
tagsToPath.put(labradorTim, null); tagsToPath.put(labradorTim, null);
db = new PdbDB(dataDirectory); db = new PdbDB(dataDirectory);
for (final Tags tags : tagsToPath.keySet()) { for (final Tags tags : tagsToPath.keySet()) {
final Path newFile = db.createNewFile(tags); final Path newFile = db.createNewFile(tags);
tagsToPath.put(tags, newFile); tagsToPath.put(tags, newFile);
} }
} }
public void testEmptyQuery() throws Exception { public void testEmptyQuery() throws Exception {
assertProposals("", 0, // assertProposals("", 0, //
new Proposal("name", "name=*", 6), // new Proposal("name", "name=*", true), //
new Proposal("bird", "bird=*", 4), // new Proposal("bird", "bird=*", true), //
new Proposal("dog", "dog=*", 2)// new Proposal("dog", "dog=*", true)//
); );
assertProposals(" ", 1, // assertProposals(" ", 1, //
new Proposal("name", "name=*", 6), // new Proposal("name", "name=*", true), //
new Proposal("bird", "bird=*", 4), // new Proposal("bird", "bird=*", true), //
new Proposal("dog", "dog=*", 2)// new Proposal("dog", "dog=*", true)//
); );
} }
public void testPrefixOfKey() throws Exception { public void testPrefixOfKey() throws Exception {
assertProposals("bi", 2, // assertProposals("bi", 2, //
new Proposal("bird", "bird=* ", 4) // new Proposal("bird", "bird=* ", true) //
); );
assertProposals("bird", 4, // assertProposals("bird", 4, //
new Proposal("bird", "bird=* ", 4) // new Proposal("bird", "bird=* ", true) //
); );
} }
public void testPrefixOfValue() throws Exception { public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim", 9, // assertProposals("name =Tim", 9, //
new Proposal("Timothy", "name =Timothy ", 1) new Proposal("Timothy", "name =Timothy ", true)
); );
/* assertProposals("name =Je", 8, //
assertProposals("name =Je", 8, // new Proposal("Jennifer", "name =Jennifer ", true), //
new Proposal("Jennifer", "name =Jennifer ", 2), // new Proposal("Jenny", "name =Jenny ", true) //
new Proposal("Jenny", "name =Jenny ", 1) // );
);
assertProposals("bird=eagle and n", 16, //
assertProposals("bird=eagle and n", 16, // new Proposal("name", "bird=eagle and name=* ", true) //
new Proposal("name", "bird=eagle and name=* ", 1) // );
); /*
*/ */
} }
private void assertProposals(final String query, final int caretIndex, final Proposal... expected) private void assertProposals(final String query, final int caretIndex, final Proposal... expected)
throws InterruptedException { throws InterruptedException {
final List<Proposal> actual = db.propose(query, caretIndex); final List<Proposal> actual = db.propose(query, caretIndex);
final List<Proposal> expectedList = Arrays.asList(expected); final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList); Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---"); System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + actual); System.out.println("actual : " + actual);
System.out.println("expected: " + expectedList); System.out.println("expected: " + expectedList);
Assert.assertEquals(expectedList, actual); Assert.assertEquals(expectedList, actual);
} }
} }

312
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -1,156 +1,156 @@
package org.lucares.pdbui; package org.lucares.pdbui;
import java.text.Collator; import java.text.Collator;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.SortedSet; import java.util.SortedSet;
import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.PlotSettings; import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdbui.domain.AutocompleteProposal; import org.lucares.pdbui.domain.AutocompleteProposal;
import org.lucares.pdbui.domain.AutocompleteProposalByValue; import org.lucares.pdbui.domain.AutocompleteProposalByValue;
import org.lucares.pdbui.domain.AutocompleteResponse; import org.lucares.pdbui.domain.AutocompleteResponse;
import org.lucares.pdbui.domain.PlotRequest; import org.lucares.pdbui.domain.PlotRequest;
import org.lucares.pdbui.domain.PlotResponse; import org.lucares.pdbui.domain.PlotResponse;
import org.lucares.performance.db.PerformanceDb; import org.lucares.performance.db.PerformanceDb;
import org.lucares.recommind.logs.DataSeries; import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.InternalPlottingException; import org.lucares.recommind.logs.InternalPlottingException;
import org.lucares.recommind.logs.NoDataPointsException; import org.lucares.recommind.logs.NoDataPointsException;
import org.lucares.recommind.logs.PlotResult; import org.lucares.recommind.logs.PlotResult;
import org.lucares.recommind.logs.Plotter; import org.lucares.recommind.logs.Plotter;
import org.lucares.utils.CollectionUtils; import org.lucares.utils.CollectionUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller; import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.ModelAndView;
@Controller @Controller
@EnableAutoConfiguration @EnableAutoConfiguration
public class PdbController implements HardcodedValues { public class PdbController implements HardcodedValues {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class); private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class);
private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00"); private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00");
private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59"); private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59");
private final Plotter plotter; private final Plotter plotter;
private final PerformanceDb db; private final PerformanceDb db;
public PdbController(final PerformanceDb db, final Plotter plotter) { public PdbController(final PerformanceDb db, final Plotter plotter) {
this.db = db; this.db = db;
this.plotter = plotter; this.plotter = plotter;
} }
@GetMapping("/") @GetMapping("/")
public ModelAndView index() { public ModelAndView index() {
final String view = "main"; final String view = "main";
final Map<String, Object> model = new HashMap<>(); final Map<String, Object> model = new HashMap<>();
model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN)); model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN));
model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END)); model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END));
return new ModelAndView(view, model); return new ModelAndView(view, model);
} }
@RequestMapping(path = "/plots", // @RequestMapping(path = "/plots", //
method = RequestMethod.POST, // method = RequestMethod.POST, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE // produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
) )
@ResponseBody @ResponseBody
PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException { PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException {
final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request); final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request);
try { try {
final PlotResult result = plotter.plot(plotSettings); final PlotResult result = plotter.plot(plotSettings);
final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName(); final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName();
LOGGER.trace("image url: {}", imageUrl); LOGGER.trace("image url: {}", imageUrl);
System.gc(); System.gc();
return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl); return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl);
} catch (final NoDataPointsException e) { } catch (final NoDataPointsException e) {
throw new NotFoundException(e); throw new NotFoundException(e);
} }
} }
@RequestMapping(path = "/autocomplete", // @RequestMapping(path = "/autocomplete", //
method = RequestMethod.GET, // method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, // consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE // produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
) )
@ResponseBody @ResponseBody
AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query, AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query,
@RequestParam(name = "caretIndex") final int caretIndex) { @RequestParam(name = "caretIndex") final int caretIndex) {
final AutocompleteResponse result = new AutocompleteResponse(); final AutocompleteResponse result = new AutocompleteResponse();
final int zeroBasedCaretIndex = caretIndex - 1; final int zeroBasedCaretIndex = caretIndex - 1;
final List<Proposal> proposals = db.autocomplete(query, zeroBasedCaretIndex); final List<Proposal> proposals = db.autocomplete(query, zeroBasedCaretIndex);
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.getResults() > 0); final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults() );
final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(nonEmptyProposals); final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(nonEmptyProposals);
Collections.sort(autocompleteProposals, new AutocompleteProposalByValue()); Collections.sort(autocompleteProposals, new AutocompleteProposalByValue());
result.setProposals(autocompleteProposals); result.setProposals(autocompleteProposals);
return result; return result;
} }
@RequestMapping(path = "/fields", // @RequestMapping(path = "/fields", //
method = RequestMethod.GET, // method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE // produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
) )
@ResponseBody @ResponseBody
List<String> fields() { List<String> fields() {
final List<String> fields = db.getFields(); final List<String> fields = db.getFields();
fields.sort(Collator.getInstance(Locale.ENGLISH)); fields.sort(Collator.getInstance(Locale.ENGLISH));
return fields; return fields;
} }
@RequestMapping(path = "/fields/{fieldName}/values", // @RequestMapping(path = "/fields/{fieldName}/values", //
method = RequestMethod.GET, // method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE // produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
) )
@ResponseBody @ResponseBody
SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName, SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) { @RequestParam(name = "query") final String query) {
final SortedSet<String> fields = db.getFieldsValues(query, fieldName); final SortedSet<String> fields = db.getFieldsValues(query, fieldName);
return fields; return fields;
} }
private List<AutocompleteProposal> toAutocompleteProposals(final List<Proposal> proposals) { private List<AutocompleteProposal> toAutocompleteProposals(final List<Proposal> proposals) {
final List<AutocompleteProposal> result = new ArrayList<>(); final List<AutocompleteProposal> result = new ArrayList<>();
for (final Proposal proposal : proposals) { for (final Proposal proposal : proposals) {
final AutocompleteProposal e = new AutocompleteProposal(); final AutocompleteProposal e = new AutocompleteProposal();
e.setValue(proposal.getProposedTag()); e.setValue(proposal.getProposedTag());
e.setProposedQuery(proposal.getProposedQuery()); e.setProposedQuery(proposal.getProposedQuery());
result.add(e); result.add(e);
} }
return result; return result;
} }
} }

1
pdb-ui/src/main/resources/log4j2.xml
View File

@@ -21,5 +21,6 @@
<logger name="org.lucares.metrics.proposals" level="DEBUG" /> <logger name="org.lucares.metrics.proposals" level="DEBUG" />
<logger name="org.lucares.metrics.plotter" level="DEBUG" /> <logger name="org.lucares.metrics.plotter" level="DEBUG" />
<logger name="org.lucares.metrics.gnuplot" level="DEBUG" /> <logger name="org.lucares.metrics.gnuplot" level="DEBUG" />
<logger name="org.lucares.metrics.dataStore" level="DEBUG" />
</Loggers> </Loggers>
</Configuration> </Configuration>