andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

do not compute counts when proposing all keys

Browse Source
This commit is contained in:
Andreas Huber
2017-11-18 13:03:45 +01:00
parent 14d1367c4a
commit a6a2236d18
7 changed files with 584 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
data-store/src/main/java/org/lucares/pdb/datastore/Proposal.java
View File

@@ -5,19 +5,19 @@ public class Proposal implements Comparable<Proposal> {
private final String proposedQuery;
private final long results;
private final boolean hasResults;
public Proposal(final String proposedTag, final String proposedQuery, final long results) {
public Proposal(final String proposedTag, final String proposedQuery, final boolean hasResults) {
super();
this.proposedTag = proposedTag;
this.proposedQuery = proposedQuery;
this.results = results;
this.hasResults = hasResults;
}
public Proposal(final Proposal proposal, final long results) {
public Proposal(final Proposal proposal, final boolean hasResults) {
this.proposedTag = proposal.proposedTag;
this.proposedQuery = proposal.proposedQuery;
this.results = results;
this.hasResults = hasResults;
}
public String getProposedTag() {
@@ -28,35 +28,42 @@ public class Proposal implements Comparable<Proposal> {
return proposedQuery;
}
public long getResults() {
return results;
public boolean hasResults() {
return hasResults;
}
@Override
public String toString() {
return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", results=" + results
return "Proposal [proposedTag:" + proposedTag + ", proposedQuery:" + proposedQuery + ", hasResults=" + hasResults
+ "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result + ((proposedTag == null) ? 0 : proposedTag.hashCode());
result = prime * result + (int) (results ^ (results >>> 32));
result = prime * result + (hasResults ? 1231 : 1237);
result = prime * result
+ ((proposedQuery == null) ? 0 : proposedQuery.hashCode());
result = prime * result
+ ((proposedTag == null) ? 0 : proposedTag.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Proposal other = (Proposal) obj;
Proposal other = (Proposal) obj;
if (hasResults != other.hasResults)
return false;
if (proposedQuery == null) {
if (other.proposedQuery != null)
return false;
@@ -67,18 +74,11 @@ public class Proposal implements Comparable<Proposal> {
return false;
} else if (!proposedTag.equals(other.proposedTag))
return false;
if (results != other.results)
return false;
return true;
}
@Override
public int compareTo(final Proposal o) {
if (results != o.results) {
return results < o.results ? 1 : -1;
}
return proposedTag.compareToIgnoreCase(o.proposedTag);
public int compareTo(Proposal o) {
return proposedTag.compareTo(o.getProposedTag());
}
}

5
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -22,8 +22,11 @@ import org.lucares.pdb.datastore.lang.Expression;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor.AllDocIds;
import org.lucares.pdb.datastore.lang.QueryLanguageParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DataStore {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final String SUBDIR_STORAGE = "storage";
private static final String PDB_EXTENSION = ".pdb";
@@ -196,10 +199,12 @@ public class DataStore {
}
private IntList executeQuery(final String query) {
final long start = System.nanoTime();
final Expression expression = QueryLanguageParser.parse(query);
final ExpressionToDocIdVisitor visitor = new ExpressionToDocIdVisitor(keyToValueToDocId,
new AllDocIds(docIdToDoc));
final IntList docIdsList = expression.visit(visitor);
EXECUTE_QUERY_LOGGER.debug("executeQuery({}) took {}ms returned {} results " , query, (System.nanoTime() - start) / 1_000_000.0, docIdsList.size());
return docIdsList;
}

12
data-store/src/main/java/org/lucares/pdb/datastore/internal/Proposer.java
View File

@@ -28,7 +28,7 @@ public class Proposer {
result = ProposerParser.parse(query, dataStore, caretIndex);
}
return CollectionUtils.filter(result, p -> p.getResults() >= 0);
return CollectionUtils.filter(result, p -> p.hasResults() );
}
private SortedSet<Proposal> proposeForAllKeys() {
@@ -37,23 +37,21 @@ public class Proposer {
final Map<String, String> fieldToQuery = CollectionUtils.createMapFromKeys(fields, f -> f + "=*");
result = computeProposalsForQueries(fieldToQuery);
result = toProposalsForQueries(fieldToQuery);
return result;
}
private SortedSet<Proposal> computeProposalsForQueries(final Map<String, String> keyToQuery) {
private SortedSet<Proposal> toProposalsForQueries(final Map<String, String> keyToQuery) {
final SortedSet<Proposal> result = new TreeSet<>();
for (final Entry<String, String> e : keyToQuery.entrySet()) {
final String key = e.getKey();
final String query = e.getValue();
final int count = dataStore.count(query);
final Proposal proposal = new Proposal(key, query, count);
final Proposal proposal = new Proposal(key, query, true);
result.add(proposal);
}
return result;
}
}

552
data-store/src/main/java/org/lucares/pdb/datastore/lang/QueryCompletionPdbLangParser.java
View File

@@ -1,276 +1,276 @@
package org.lucares.pdb.datastore.lang;
import java.util.BitSet;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.CollectionUtils;
public class QueryCompletionPdbLangParser extends PdbLangParser {
public class Listener implements PdbLangListener, ANTLRErrorListener {
private final int caretPosition;
private final DataStore dataStore;
private final SortedSet<Proposal> proposals = new TreeSet<>();
private final String query;
public Listener(final String query, final DataStore dataStore, final int caretPosition) {
this.query = query;
this.dataStore = dataStore;
this.caretPosition = caretPosition;
}
public SortedSet<Proposal> getProposals() {
return proposals;
}
@Override
public void visitTerminal(final TerminalNode node) {
if (containsCaret(node) && !isEOF(node)) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
if (_ctx instanceof PropertyTerminalExpressionContext) {
final String propertyKey = _ctx.getParent().children.get(0).getText();
final String propertyValuePrefix = node.getText().substring(0, caretPosition - start);
final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey,
propertyValuePrefix);
proposedValues.stream()//
.map(v -> {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(start, end + 1, v + " ");
return new Proposal(v, newQuery.toString(), -1);
}).map(p -> {
final int count = dataStore.count(p.getProposedQuery());
return new Proposal(p, count);
}).forEach(proposals::add);
} else if (_ctx instanceof IdentifierExpressionContext) {
final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start);
final StringBuilder newQueryPattern = new StringBuilder(query);
newQueryPattern.replace(start, end + 1, "%s");
addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString());
}
}
}
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) {
final CommonToken token = (CommonToken) offendingSymbol;
final String text = token.getText();
if ("and".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and ");
proposals.add(new Proposal(" and ", newQuery.toString(), 1));
}
if ("or".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or ");
proposals.add(new Proposal(" or ", newQuery.toString(), 1));
}
}
}
private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) {
final List<String> availableKeys = dataStore.getAvailableFields();
final List<String> matchingKeys = CollectionUtils.filter(availableKeys,
s -> s.startsWith(propertyKeyPrefix));
matchingKeys.stream()//
.map(key -> {
return new Proposal(key, String.format(newQueryPattern, key + "=* "), -1);
}).map(p -> {
final String proposedQuery = p.getProposedQuery();
final int count = count(proposedQuery);
return new Proposal(p, count);
}).forEach(proposals::add);
}
private int count(final String proposedQuery) {
try {
return dataStore.count(proposedQuery);
} catch (final SyntaxException e) {
return -1;
}
}
private boolean isEOF(final Object offendingSymbol) {
if (offendingSymbol instanceof CommonToken) {
return ((CommonToken) offendingSymbol).getType() < 0;
}
return false;
}
@Override
public void visitErrorNode(final ErrorNode node) {
}
@Override
public void enterEveryRule(final ParserRuleContext ctx) {
}
@Override
public void exitEveryRule(final ParserRuleContext ctx) {
}
@Override
public void enterStart(final StartContext ctx) {
}
@Override
public void exitStart(final StartContext ctx) {
}
@Override
public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void enterNotExpression(final NotExpressionContext ctx) {
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
}
@Override
public void enterParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void exitParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void enterPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void enterIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
// if (containsCaret(ctx)) {
// final int start = ctx.getStart().getStartIndex();
// final int end = ctx.getStop().getStopIndex();
// final int ruleIndex = _ctx.getRuleIndex();
//
// final String prefix = ctx.getText().substring(0, caretPosition -
// start);
// ctx.getParent().children.get(0).getText();
//
// proposals.addAll(getPropertyValuesByPrefix(prefix));
// }
}
private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey,
final String propertyValuePrefix) {
final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey);
final SortedSet<String> result = new TreeSet<>();
for (final String value : availableValuesForKey) {
if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) {
result.add(value);
}
}
return result;
}
@Override
public void enterEqual(final EqualContext ctx) {
}
@Override
public void exitEqual(final EqualContext ctx) {
}
private boolean isEOF(final TerminalNode node) {
return node.getSymbol().getType() < 0;
}
private boolean containsCaret(final TerminalNode node) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
return start <= caretPosition && end + 1 >= caretPosition;
}
@Override
public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex,
final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) {
}
@Override
public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) {
}
@Override
public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final int prediction, final ATNConfigSet configs) {
}
}
public QueryCompletionPdbLangParser(final TokenStream input) {
super(input);
}
}
package org.lucares.pdb.datastore.lang;
import java.util.BitSet;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
import org.antlr.v4.runtime.ANTLRErrorListener;
import org.antlr.v4.runtime.CommonToken;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.atn.ATNConfigSet;
import org.antlr.v4.runtime.dfa.DFA;
import org.antlr.v4.runtime.tree.ErrorNode;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.CollectionUtils;
public class QueryCompletionPdbLangParser extends PdbLangParser {
public class Listener implements PdbLangListener, ANTLRErrorListener {
private final int caretPosition;
private final DataStore dataStore;
private final SortedSet<Proposal> proposals = new TreeSet<>();
private final String query;
public Listener(final String query, final DataStore dataStore, final int caretPosition) {
this.query = query;
this.dataStore = dataStore;
this.caretPosition = caretPosition;
}
public SortedSet<Proposal> getProposals() {
return proposals;
}
@Override
public void visitTerminal(final TerminalNode node) {
if (containsCaret(node) && !isEOF(node)) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
if (_ctx instanceof PropertyTerminalExpressionContext) {
final String propertyKey = _ctx.getParent().children.get(0).getText();
final String propertyValuePrefix = node.getText().substring(0, caretPosition - start);
final SortedSet<String> proposedValues = getPropertyValuesByPrefix(propertyKey,
propertyValuePrefix);
proposedValues.stream()//
.map(v -> {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(start, end + 1, v + " "); // insert the terminal into the query
return new Proposal(v, newQuery.toString(), false);
}).map(p -> {
final int count = dataStore.count(p.getProposedQuery());
return new Proposal(p, count> 0);
}).forEach(proposals::add);
} else if (_ctx instanceof IdentifierExpressionContext) {
final String propertyKeyPrefix = node.getText().substring(0, caretPosition - start);
final StringBuilder newQueryPattern = new StringBuilder(query);
newQueryPattern.replace(start, end + 1, "%s");
addProposalsForKeys(propertyKeyPrefix, newQueryPattern.toString());
}
}
}
@Override
public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
final int charPositionInLine, final String msg, final RecognitionException e) {
if (!isEOF(offendingSymbol) && offendingSymbol instanceof CommonToken) {
final CommonToken token = (CommonToken) offendingSymbol;
final String text = token.getText();
if ("and".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " and ");
proposals.add(new Proposal(" and ", newQuery.toString(), true));
}
if ("or".startsWith(text)) {
final StringBuilder newQuery = new StringBuilder(query);
newQuery.replace(charPositionInLine, charPositionInLine + text.length(), " or ");
proposals.add(new Proposal(" or ", newQuery.toString(), true));
}
}
}
private void addProposalsForKeys(final String propertyKeyPrefix, final String newQueryPattern) {
final List<String> availableKeys = dataStore.getAvailableFields();
final List<String> matchingKeys = CollectionUtils.filter(availableKeys,
s -> s.startsWith(propertyKeyPrefix));
matchingKeys.stream()//
.map(key -> {
return new Proposal(key, String.format(newQueryPattern, key + "=* "), false);
}).map(p -> {
final String proposedQuery = p.getProposedQuery();
final int count = count(proposedQuery);
return new Proposal(p, count > 0);
}).forEach(proposals::add);
}
private int count(final String proposedQuery) {
try {
return dataStore.count(proposedQuery);
} catch (final SyntaxException e) {
return -1;
}
}
private boolean isEOF(final Object offendingSymbol) {
if (offendingSymbol instanceof CommonToken) {
return ((CommonToken) offendingSymbol).getType() < 0;
}
return false;
}
@Override
public void visitErrorNode(final ErrorNode node) {
}
@Override
public void enterEveryRule(final ParserRuleContext ctx) {
}
@Override
public void exitEveryRule(final ParserRuleContext ctx) {
}
@Override
public void enterStart(final StartContext ctx) {
}
@Override
public void exitStart(final StartContext ctx) {
}
@Override
public void enterBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void exitBinaryOrExpression(final BinaryOrExpressionContext ctx) {
}
@Override
public void enterBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void exitBinaryAndExpression(final BinaryAndExpressionContext ctx) {
}
@Override
public void enterNotExpression(final NotExpressionContext ctx) {
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
}
@Override
public void enterParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void exitParenExpression(final ParenExpressionContext ctx) {
}
@Override
public void enterPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) {
}
@Override
public void enterIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
}
@Override
public void enterPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
}
@Override
public void exitPropertyTerminalExpression(final PropertyTerminalExpressionContext ctx) {
// if (containsCaret(ctx)) {
// final int start = ctx.getStart().getStartIndex();
// final int end = ctx.getStop().getStopIndex();
// final int ruleIndex = _ctx.getRuleIndex();
//
// final String prefix = ctx.getText().substring(0, caretPosition -
// start);
// ctx.getParent().children.get(0).getText();
//
// proposals.addAll(getPropertyValuesByPrefix(prefix));
// }
}
private SortedSet<String> getPropertyValuesByPrefix(final String propertyKey,
final String propertyValuePrefix) {
final SortedSet<String> availableValuesForKey = dataStore.getAvailableValuesForKey("", propertyKey);
final SortedSet<String> result = new TreeSet<>();
for (final String value : availableValuesForKey) {
if (value.startsWith(propertyValuePrefix) && !value.equals(propertyValuePrefix)) {
result.add(value);
}
}
return result;
}
@Override
public void enterEqual(final EqualContext ctx) {
}
@Override
public void exitEqual(final EqualContext ctx) {
}
private boolean isEOF(final TerminalNode node) {
return node.getSymbol().getType() < 0;
}
private boolean containsCaret(final TerminalNode node) {
final int start = node.getSymbol().getStartIndex();
final int end = node.getSymbol().getStopIndex();
return start <= caretPosition && end + 1 >= caretPosition;
}
@Override
public void reportAmbiguity(final Parser recognizer, final DFA dfa, final int startIndex, final int stopIndex,
final boolean exact, final BitSet ambigAlts, final ATNConfigSet configs) {
}
@Override
public void reportAttemptingFullContext(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final BitSet conflictingAlts, final ATNConfigSet configs) {
}
@Override
public void reportContextSensitivity(final Parser recognizer, final DFA dfa, final int startIndex,
final int stopIndex, final int prediction, final ATNConfigSet configs) {
}
}
public QueryCompletionPdbLangParser(final TokenStream input) {
super(input);
}
}

238
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -1,119 +1,119 @@
package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbDB;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test
public class ProposerTest {
private Path dataDirectory;
private PdbDB db;
private Map<Tags, Path> tagsToPath;
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
db = null;
tagsToPath = null;
}
private void initDatabase() throws Exception {
tagsToPath = new LinkedHashMap<>();
final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim");
tagsToPath.put(eagleTim, null);
tagsToPath.put(eagleTimothy, null);
tagsToPath.put(pigeonJennifer, null);
tagsToPath.put(flamingoJennifer, null);
tagsToPath.put(labradorJenny, null);
tagsToPath.put(labradorTim, null);
db = new PdbDB(dataDirectory);
for (final Tags tags : tagsToPath.keySet()) {
final Path newFile = db.createNewFile(tags);
tagsToPath.put(tags, newFile);
}
}
public void testEmptyQuery() throws Exception {
assertProposals("", 0, //
new Proposal("name", "name=*", 6), //
new Proposal("bird", "bird=*", 4), //
new Proposal("dog", "dog=*", 2)//
);
assertProposals(" ", 1, //
new Proposal("name", "name=*", 6), //
new Proposal("bird", "bird=*", 4), //
new Proposal("dog", "dog=*", 2)//
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi", 2, //
new Proposal("bird", "bird=* ", 4) //
);
assertProposals("bird", 4, //
new Proposal("bird", "bird=* ", 4) //
);
}
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim", 9, //
new Proposal("Timothy", "name =Timothy ", 1)
);
/*
assertProposals("name =Je", 8, //
new Proposal("Jennifer", "name =Jennifer ", 2), //
new Proposal("Jenny", "name =Jenny ", 1) //
);
assertProposals("bird=eagle and n", 16, //
new Proposal("name", "bird=eagle and name=* ", 1) //
);
*/
}
private void assertProposals(final String query, final int caretIndex, final Proposal... expected)
throws InterruptedException {
final List<Proposal> actual = db.propose(query, caretIndex);
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + actual);
System.out.println("expected: " + expectedList);
Assert.assertEquals(expectedList, actual);
}
}
package org.lucares.pdb.datastore.internal;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbDB;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@Test
public class ProposerTest {
private Path dataDirectory;
private PdbDB db;
private Map<Tags, Path> tagsToPath;
@BeforeClass
public void beforeClass() throws Exception {
dataDirectory = Files.createTempDirectory("pdb");
initDatabase();
}
@AfterClass
public void afterClass() throws IOException {
FileUtils.delete(dataDirectory);
db = null;
tagsToPath = null;
}
private void initDatabase() throws Exception {
tagsToPath = new LinkedHashMap<>();
final Tags eagleTim = Tags.create("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.create("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.create("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.create("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.create("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.create("dog", "labrador", "name", "Tim");
tagsToPath.put(eagleTim, null);
tagsToPath.put(eagleTimothy, null);
tagsToPath.put(pigeonJennifer, null);
tagsToPath.put(flamingoJennifer, null);
tagsToPath.put(labradorJenny, null);
tagsToPath.put(labradorTim, null);
db = new PdbDB(dataDirectory);
for (final Tags tags : tagsToPath.keySet()) {
final Path newFile = db.createNewFile(tags);
tagsToPath.put(tags, newFile);
}
}
public void testEmptyQuery() throws Exception {
assertProposals("", 0, //
new Proposal("name", "name=*", true), //
new Proposal("bird", "bird=*", true), //
new Proposal("dog", "dog=*", true)//
);
assertProposals(" ", 1, //
new Proposal("name", "name=*", true), //
new Proposal("bird", "bird=*", true), //
new Proposal("dog", "dog=*", true)//
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi", 2, //
new Proposal("bird", "bird=* ", true) //
);
assertProposals("bird", 4, //
new Proposal("bird", "bird=* ", true) //
);
}
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim", 9, //
new Proposal("Timothy", "name =Timothy ", true)
);
assertProposals("name =Je", 8, //
new Proposal("Jennifer", "name =Jennifer ", true), //
new Proposal("Jenny", "name =Jenny ", true) //
);
assertProposals("bird=eagle and n", 16, //
new Proposal("name", "bird=eagle and name=* ", true) //
);
/*
*/
}
private void assertProposals(final String query, final int caretIndex, final Proposal... expected)
throws InterruptedException {
final List<Proposal> actual = db.propose(query, caretIndex);
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);
System.out.println("\n\n--- " + query + " ---");
System.out.println("actual : " + actual);
System.out.println("expected: " + expectedList);
Assert.assertEquals(expectedList, actual);
}
}

312
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -1,156 +1,156 @@
package org.lucares.pdbui;
import java.text.Collator;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedSet;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdbui.domain.AutocompleteProposal;
import org.lucares.pdbui.domain.AutocompleteProposalByValue;
import org.lucares.pdbui.domain.AutocompleteResponse;
import org.lucares.pdbui.domain.PlotRequest;
import org.lucares.pdbui.domain.PlotResponse;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.InternalPlottingException;
import org.lucares.recommind.logs.NoDataPointsException;
import org.lucares.recommind.logs.PlotResult;
import org.lucares.recommind.logs.Plotter;
import org.lucares.utils.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
@Controller
@EnableAutoConfiguration
public class PdbController implements HardcodedValues {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class);
private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00");
private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59");
private final Plotter plotter;
private final PerformanceDb db;
public PdbController(final PerformanceDb db, final Plotter plotter) {
this.db = db;
this.plotter = plotter;
}
@GetMapping("/")
public ModelAndView index() {
final String view = "main";
final Map<String, Object> model = new HashMap<>();
model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN));
model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END));
return new ModelAndView(view, model);
}
@RequestMapping(path = "/plots", //
method = RequestMethod.POST, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException {
final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request);
try {
final PlotResult result = plotter.plot(plotSettings);
final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName();
LOGGER.trace("image url: {}", imageUrl);
System.gc();
return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl);
} catch (final NoDataPointsException e) {
throw new NotFoundException(e);
}
}
@RequestMapping(path = "/autocomplete", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query,
@RequestParam(name = "caretIndex") final int caretIndex) {
final AutocompleteResponse result = new AutocompleteResponse();
final int zeroBasedCaretIndex = caretIndex - 1;
final List<Proposal> proposals = db.autocomplete(query, zeroBasedCaretIndex);
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.getResults() > 0);
final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(nonEmptyProposals);
Collections.sort(autocompleteProposals, new AutocompleteProposalByValue());
result.setProposals(autocompleteProposals);
return result;
}
@RequestMapping(path = "/fields", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
List<String> fields() {
final List<String> fields = db.getFields();
fields.sort(Collator.getInstance(Locale.ENGLISH));
return fields;
}
@RequestMapping(path = "/fields/{fieldName}/values", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) {
final SortedSet<String> fields = db.getFieldsValues(query, fieldName);
return fields;
}
private List<AutocompleteProposal> toAutocompleteProposals(final List<Proposal> proposals) {
final List<AutocompleteProposal> result = new ArrayList<>();
for (final Proposal proposal : proposals) {
final AutocompleteProposal e = new AutocompleteProposal();
e.setValue(proposal.getProposedTag());
e.setProposedQuery(proposal.getProposedQuery());
result.add(e);
}
return result;
}
}
package org.lucares.pdbui;
import java.text.Collator;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedSet;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.PlotSettings;
import org.lucares.pdbui.domain.AutocompleteProposal;
import org.lucares.pdbui.domain.AutocompleteProposalByValue;
import org.lucares.pdbui.domain.AutocompleteResponse;
import org.lucares.pdbui.domain.PlotRequest;
import org.lucares.pdbui.domain.PlotResponse;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.recommind.logs.DataSeries;
import org.lucares.recommind.logs.InternalPlottingException;
import org.lucares.recommind.logs.NoDataPointsException;
import org.lucares.recommind.logs.PlotResult;
import org.lucares.recommind.logs.Plotter;
import org.lucares.utils.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
@Controller
@EnableAutoConfiguration
public class PdbController implements HardcodedValues {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbController.class);
private static final DateTimeFormatter DATE_FORMAT_BEGIN = DateTimeFormatter.ofPattern("yyyy-MM-dd 00:00:00");
private static final DateTimeFormatter DATE_FORMAT_END = DateTimeFormatter.ofPattern("yyyy-MM-dd 23:59:59");
private final Plotter plotter;
private final PerformanceDb db;
public PdbController(final PerformanceDb db, final Plotter plotter) {
this.db = db;
this.plotter = plotter;
}
@GetMapping("/")
public ModelAndView index() {
final String view = "main";
final Map<String, Object> model = new HashMap<>();
model.put("oldestValue", LocalDateTime.now().minusDays(7).format(DATE_FORMAT_BEGIN));
model.put("latestValue", LocalDateTime.now().format(DATE_FORMAT_END));
return new ModelAndView(view, model);
}
@RequestMapping(path = "/plots", //
method = RequestMethod.POST, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
PlotResponse createPlot(@RequestBody final PlotRequest request) throws InternalPlottingException {
final PlotSettings plotSettings = PlotSettingsTransformer.toSettings(request);
try {
final PlotResult result = plotter.plot(plotSettings);
final String imageUrl = WEB_IMAGE_OUTPUT_PATH + "/" + result.getImageName();
LOGGER.trace("image url: {}", imageUrl);
System.gc();
return new PlotResponse(DataSeries.toMap(result.getDataSeries()), imageUrl);
} catch (final NoDataPointsException e) {
throw new NotFoundException(e);
}
}
@RequestMapping(path = "/autocomplete", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
AutocompleteResponse autocomplete(@RequestParam(name = "query") final String query,
@RequestParam(name = "caretIndex") final int caretIndex) {
final AutocompleteResponse result = new AutocompleteResponse();
final int zeroBasedCaretIndex = caretIndex - 1;
final List<Proposal> proposals = db.autocomplete(query, zeroBasedCaretIndex);
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults() );
final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(nonEmptyProposals);
Collections.sort(autocompleteProposals, new AutocompleteProposalByValue());
result.setProposals(autocompleteProposals);
return result;
}
@RequestMapping(path = "/fields", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
List<String> fields() {
final List<String> fields = db.getFields();
fields.sort(Collator.getInstance(Locale.ENGLISH));
return fields;
}
@RequestMapping(path = "/fields/{fieldName}/values", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) {
final SortedSet<String> fields = db.getFieldsValues(query, fieldName);
return fields;
}
private List<AutocompleteProposal> toAutocompleteProposals(final List<Proposal> proposals) {
final List<AutocompleteProposal> result = new ArrayList<>();
for (final Proposal proposal : proposals) {
final AutocompleteProposal e = new AutocompleteProposal();
e.setValue(proposal.getProposedTag());
e.setProposedQuery(proposal.getProposedQuery());
result.add(e);
}
return result;
}
}

1
pdb-ui/src/main/resources/log4j2.xml
View File

@@ -21,5 +21,6 @@
<logger name="org.lucares.metrics.proposals" level="DEBUG" />
<logger name="org.lucares.metrics.plotter" level="DEBUG" />
<logger name="org.lucares.metrics.gnuplot" level="DEBUG" />
<logger name="org.lucares.metrics.dataStore" level="DEBUG" />
</Loggers>
</Configuration>