andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

test for keywords db performance

Browse Source
This commit is contained in:
Andreas Huber
2017-01-07 09:10:42 +01:00
parent c283568757
commit 4f77515bbd
20 changed files with 1211 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pdb-keyword-db/.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
/bin/
/build/
/.settings/
/.classpath
/.project
/test-output/

17
pdb-keyword-db/build.gradle Normal file
View File

@@ -0,0 +1,17 @@
apply plugin: 'antlr'
dependencies {
compile 'org.apache.commons:commons-collections4:4.1'
runtime "org.antlr:antlr4:4.5.3"
antlr "org.antlr:antlr4:4.5.3" // use ANTLR version 4
}
sourceSets {
generated{
java.srcDir "build/generated-src/antlr/main"
}
}
compileJava{
source += sourceSets.generated.java
}

62
pdb-keyword-db/src/main/antlr/org/lucares/pdb/keyword/db/KeywordsLang.g4 Normal file
View File

@@ -0,0 +1,62 @@
grammar KeywordsLang;
@header {
package org.lucares.pdb.keyword.db;
}
start : expression EOF ;
expression
: prop=identifier eq=equal value=identifier #propertyExpression
| LPAREN expression RPAREN #parenExpression
| NOT expression #notExpression
| left=expression op=and right=expression #andExpression
| left=expression op=or right=expression #orExpression
;
identifier
: IDENTIFIER #identifierExpression
;
and : AND ;
or : OR ;
equal : EQUAL ;
AND : [aA][nN][dD] ;
OR : [oO][rR] ;
NOT : '!';
EQUAL : '=' ;
LPAREN : '(' ;
RPAREN : ')' ;
WS : [ \r\t\u000C\n]+ -> skip;
IDENTIFIER
: JavaLetter JavaLetterOrDigit*
;
fragment
JavaLetter
: [a-zA-Z$_] // these are the "java letters" below 0x7F
| [\u002a] // asterisk, used for wildcards
| // covers all characters above 0x7F which are not a surrogate
~[\u0000-\u007F\uD800-\uDBFF]
{Character.isJavaIdentifierStart(_input.LA(-1))}?
| // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
[\uD800-\uDBFF] [\uDC00-\uDFFF]
{Character.isJavaIdentifierStart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
;
fragment
JavaLetterOrDigit
: [a-zA-Z0-9$_] // these are the "java letters or digits" below 0x7F
| [\u002a] // asterisk, used for wildcards
| '.'
| '/'
| // covers all characters above 0x7F which are not a surrogate
~[\u0000-\u007F\uD800-\uDBFF]
{Character.isJavaIdentifierPart(_input.LA(-1))}?
| // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
[\uD800-\uDBFF] [\uDC00-\uDFFF]
{Character.isJavaIdentifierPart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
;

447
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/Expression.java Normal file
View File

@@ -0,0 +1,447 @@
package org.lucares.pdb.keyword.db;
abstract public class Expression {
public <T> T visit(final ExpressionVisitor<T> visitor) {
throw new UnsupportedOperationException();
}
abstract static class TemporaryExpression extends Expression {
abstract Expression toExpression(Expression left, Expression right);
}
static class OrTemporary extends TemporaryExpression {
@Override
Expression toExpression(final Expression left, final Expression right) {
return new Or(left, right);
}
@Override
public String toString() {
return "OrTemporary";
}
}
static class AndTemporary extends TemporaryExpression {
@Override
Expression toExpression(final Expression left, final Expression right) {
return new And(left, right);
}
@Override
public String toString() {
return "AndTemporary";
}
}
public static Or or(final Expression left, final Expression right) {
return new Or(left, right);
}
public static Or or(final String left, final String right) {
return new Or(term(left), term(right));
}
public static And and(final Expression left, final Expression right) {
return new And(left, right);
}
public static And and(final String left, final String right) {
return new And(term(left), term(right));
}
public static Terminal term(final String value) {
return new Terminal(value);
}
public static MatchAll matchAll() {
return MatchAll.INSTANCE;
}
public static Not not(final Expression expression) {
return new Not(expression);
}
public static Not not(final String expression) {
return new Not(term(expression));
}
public static Property property(final String property, final String value) {
return new Property(property, value);
}
public static Parentheses parentheses(final Expression expression) {
return new Parentheses(expression);
}
static class Not extends Expression {
private final Expression expression;
Not(final Expression expression) {
this.expression = expression;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
Expression getExpression() {
return expression;
}
@Override
public String toString() {
return "!" + expression;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((expression == null) ? 0 : expression.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final Not other = (Not) obj;
if (expression == null) {
if (other.expression != null) {
return false;
}
} else if (!expression.equals(other.expression)) {
return false;
}
return true;
}
}
static class Or extends Expression {
private final Expression left;
private final Expression right;
Or(final Expression left, final Expression right) {
this.left = left;
this.right = right;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
Expression getLeft() {
return left;
}
Expression getRight() {
return right;
}
@Override
public String toString() {
return " (" + left + " or " + right + ") ";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((left == null) ? 0 : left.hashCode());
result = prime * result + ((right == null) ? 0 : right.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final Or other = (Or) obj;
if (left == null) {
if (other.left != null) {
return false;
}
} else if (!left.equals(other.left)) {
return false;
}
if (right == null) {
if (other.right != null) {
return false;
}
} else if (!right.equals(other.right)) {
return false;
}
return true;
}
}
static class And extends Expression {
private final Expression left;
private final Expression right;
And(final Expression left, final Expression right) {
this.left = left;
this.right = right;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
Expression getLeft() {
return left;
}
Expression getRight() {
return right;
}
@Override
public String toString() {
return " (" + left + " and " + right + ") ";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((left == null) ? 0 : left.hashCode());
result = prime * result + ((right == null) ? 0 : right.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final And other = (And) obj;
if (left == null) {
if (other.left != null) {
return false;
}
} else if (!left.equals(other.left)) {
return false;
}
if (right == null) {
if (other.right != null) {
return false;
}
} else if (!right.equals(other.right)) {
return false;
}
return true;
}
}
static class MatchAll extends Expression {
public static final MatchAll INSTANCE = new MatchAll();
private MatchAll() {
//
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
return "1=1";
}
}
static class Terminal extends Expression {
private final String value;
Terminal(final String value) {
this.value = value;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
return value;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((value == null) ? 0 : value.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final Terminal other = (Terminal) obj;
if (value == null) {
if (other.value != null) {
return false;
}
} else if (!value.equals(other.value)) {
return false;
}
return true;
}
public String getValue() {
return value;
}
}
static class Property extends Expression {
final String property;
final String stringValue;
public Property(final String property, final String value) {
this.property = property;
this.stringValue = value;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
@Override
public String toString() {
return " " + property + " = " + stringValue + " ";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((property == null) ? 0 : property.hashCode());
result = prime * result + ((stringValue == null) ? 0 : stringValue.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Property other = (Property) obj;
if (property == null) {
if (other.property != null)
return false;
} else if (!property.equals(other.property))
return false;
if (stringValue == null) {
if (other.stringValue != null)
return false;
} else if (!stringValue.equals(other.stringValue))
return false;
return true;
}
}
static class Parentheses extends Expression {
private final Expression expression;
Parentheses(final Expression expression) {
this.expression = expression;
}
@Override
public <T> T visit(final ExpressionVisitor<T> visitor) {
return visitor.visit(this);
}
public Expression getExpression() {
return expression;
}
@Override
public String toString() {
return " [ " + expression + " ] ";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((expression == null) ? 0 : expression.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final Parentheses other = (Parentheses) obj;
if (expression == null) {
if (other.expression != null) {
return false;
}
} else if (!expression.equals(other.expression)) {
return false;
}
return true;
}
}
}

92
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/ExpressionToFilesVisitor.java Normal file
View File

@@ -0,0 +1,92 @@
package org.lucares.pdb.keyword.db;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import org.lucares.pdb.keyword.db.Expression.And;
import org.lucares.pdb.keyword.db.Expression.Not;
import org.lucares.pdb.keyword.db.Expression.Or;
import org.lucares.pdb.keyword.db.Expression.Property;
public class ExpressionToFilesVisitor extends ExpressionVisitor<Set<String>> {
private static final Set<String> EMPTY = new TreeSet<>();
private final Map<String, Map<String, SortedSet<String>>> tagToFiles;
private final Map<String, Map<String, String>> fileToTags;
public ExpressionToFilesVisitor(final Map<String, Map<String, SortedSet<String>>> tagToFiles,
final Map<String, Map<String, String>> fileToTags) {
this.tagToFiles = tagToFiles;
this.fileToTags = fileToTags;
}
@Override
public Set<String> visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Set<String> leftFiles = left.visit(this);
final Set<String> rightFiles = right.visit(this);
final Set<String> result = new HashSet<>(leftFiles);
result.retainAll(rightFiles);
return result;
}
@Override
public Set<String> visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
final Set<String> leftFiles = left.visit(this);
final Set<String> rightFiles = right.visit(this);
final Set<String> result = new HashSet<>(leftFiles);
result.addAll(rightFiles);
return result;
}
@Override
public Set<String> visit(final Not expression) {
final Expression negatedExpression = expression.getExpression();
final Set<String> files = negatedExpression.visit(this);
final Set<String> result = new HashSet<>(fileToTags.keySet());
result.removeAll(files);
return result;
}
@Override
public Set<String> visit(final Expression.MatchAll expression) {
return fileToTags.keySet();
}
@Override
public Set<String> visit(final Property expression) {
final Set<String> result;
final String property = expression.property;
final String stringValue = expression.stringValue;
final Map<String, SortedSet<String>> values = tagToFiles.get(property);
if (values != null) {
final SortedSet<String> files = values.get(stringValue);
if (files != null) {
result = files;
} else {
result = EMPTY;
}
} else {
result = EMPTY;
}
return result;
}
}

31
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/ExpressionVisitor.java Normal file
View File

@@ -0,0 +1,31 @@
package org.lucares.pdb.keyword.db;
public abstract class ExpressionVisitor<T> {
public T visit(final Expression.And expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Or expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Not expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Property expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Terminal expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.MatchAll expression) {
throw new UnsupportedOperationException();
}
public T visit(final Expression.Parentheses parentheses) {
throw new UnsupportedOperationException();
}
}

57
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/Keywords.java Normal file
View File

@@ -0,0 +1,57 @@
package org.lucares.pdb.keyword.db;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
public class Keywords {
final Map<String, Map<String, String>> fileToTags = new HashMap<>();
final Map<String, Map<String, SortedSet<String>>> tagToFiles = new HashMap<>();
public void addFile(final String file, final Map<String, String> tags) {
fileToTags.put(file, tags);
for (final Entry<String, String> e : tags.entrySet()) {
final String field = e.getKey();
final String value = e.getValue();
tagToFiles.putIfAbsent(field, new HashMap<>());
final Map<String, SortedSet<String>> fieldToFiles = tagToFiles.get(field);
fieldToFiles.putIfAbsent(value, new TreeSet<>());
final SortedSet<String> files = fieldToFiles.get(value);
files.add(file);
}
}
public Collection<String> search(final String query) {
final long start = System.nanoTime();
final Expression expression = KeywordsLanguageParser.parse(query);
long duration = System.nanoTime() - start;
final String parsing = "parsing: " + duration / 1_000_000.0 + "ms";
// System.out.println(expression.visit(new PrintExpressionVisitor()));
final ExpressionToFilesVisitor visitor = new ExpressionToFilesVisitor(tagToFiles, fileToTags);
final long start2 = System.nanoTime();
final Set<String> result = expression.visit(visitor);
long duration2 = System.nanoTime() - start2;
System.out.println(
parsing + "; searching: " + duration2 / 1_000_000.0 + "ms; found=" + result.size());
return result;
}
@Override
public String toString() {
return "Keywords [files=" + fileToTags.size() + ", fields=" + tagToFiles.size() + "]";
}
}

122
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/KeywordsLangLanguage.java Normal file
View File

@@ -0,0 +1,122 @@
package org.lucares.pdb.keyword.db;
import java.util.Arrays;
import java.util.List;
import java.util.Stack;
import org.antlr.v4.gui.TreeViewer;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DiagnosticErrorListener;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.lucares.pdb.keyword.db.Expression.And;
import org.lucares.pdb.keyword.db.Expression.Not;
import org.lucares.pdb.keyword.db.Expression.Or;
import org.lucares.pdb.keyword.db.Expression.Terminal;
import org.lucares.pdb.keyword.db.KeywordsLangParser.AndExpressionContext;
import org.lucares.pdb.keyword.db.KeywordsLangParser.IdentifierExpressionContext;
import org.lucares.pdb.keyword.db.KeywordsLangParser.NotExpressionContext;
import org.lucares.pdb.keyword.db.KeywordsLangParser.OrExpressionContext;
import org.lucares.pdb.keyword.db.KeywordsLangParser.PropertyExpressionContext;
public class KeywordsLangLanguage {
public Expression parse(final String input) {
// define the input
final ANTLRInputStream in = new ANTLRInputStream(input);
// create lexer and parser
final KeywordsLangLexer lexer = new KeywordsLangLexer(in);
lexer.addErrorListener(new DiagnosticErrorListener());
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final KeywordsLangParser parser = new KeywordsLangParser(tokens);
final Stack<Expression> stack = new Stack<>();
// define a listener that is called for every terminals and
// non-terminals
final ParseTreeListener listener = new KeywordsLangBaseListener() {
@Override
public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
// System.out.println("push identifier " + ctx.getText());
if (ctx.getText().length() > 255) {
throw new SyntaxException(ctx, "token too long");
}
stack.push(new Terminal(ctx.getText()));
}
@Override
public void exitPropertyExpression(final PropertyExpressionContext ctx) {
// System.out.println("property expression");
final Expression value = stack.pop();
final Terminal property = (Terminal) stack.pop();
if (value instanceof Terminal) {
stack.push(new Expression.Property(property.getValue(), ((Terminal) value).getValue()));
} else {
throw new UnsupportedOperationException();
}
}
@Override
public void exitNotExpression(final NotExpressionContext ctx) {
final Expression expression = stack.pop();
final Expression notExpression = new Not(expression);
stack.push(notExpression);
}
@Override
public void exitAndExpression(final AndExpressionContext ctx) {
final Expression right = stack.pop();
final Expression left = stack.pop();
stack.push(new And(left, right));
}
@Override
public void exitOrExpression(final OrExpressionContext ctx) {
final Expression right = stack.pop();
final Expression left = stack.pop();
stack.push(new Or(left, right));
}
};
// Specify our entry point
final ParseTree parseTree = parser.start();
// Walk it and attach our listener
final ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(listener, parseTree);
if (stack.size() != 1) {
throw new RuntimeException("stack should have exactly one element " + stack);
}
return stack.pop();
}
public static void main(final String[] args) {
final org.antlr.v4.runtime.CharStream stream = new ANTLRInputStream("prop=value");
final KeywordsLangLexer lexer = new KeywordsLangLexer(stream);
final CommonTokenStream tokens = new CommonTokenStream(lexer);
final KeywordsLangParser parser = new KeywordsLangParser(tokens);
final ParseTree tree = parser.start();
final List<String> ruleNames = Arrays.asList(KeywordsLangParser.ruleNames);
final TreeViewer view = new TreeViewer(ruleNames, tree);
view.open();
}
}

15
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/KeywordsLanguageParser.java Normal file
View File

@@ -0,0 +1,15 @@
package org.lucares.pdb.keyword.db;
public class KeywordsLanguageParser {
public static Expression parse(final String query) {
final Expression result;
if (query == null || query.length() == 0) {
result = Expression.matchAll();
} else {
final KeywordsLangLanguage lang = new KeywordsLangLanguage();
result = lang.parse(query);
}
return result;
}
}

53
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/PrintExpressionVisitor.java Normal file
View File

@@ -0,0 +1,53 @@
package org.lucares.pdb.keyword.db;
import org.lucares.pdb.keyword.db.Expression.And;
import org.lucares.pdb.keyword.db.Expression.Not;
import org.lucares.pdb.keyword.db.Expression.Or;
import org.lucares.pdb.keyword.db.Expression.Property;
public class PrintExpressionVisitor extends ExpressionVisitor<String> {
public PrintExpressionVisitor() {
}
@Override
public String visit(final And expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
return "(" + left.visit(this) + " and " + right.visit(this) + ")";
}
@Override
public String visit(final Or expression) {
final Expression left = expression.getLeft();
final Expression right = expression.getRight();
return "(" + left.visit(this) + " or " + right.visit(this) + ")";
}
@Override
public String visit(final Not expression) {
final Expression negatedExpression = expression.getExpression();
return "!" + negatedExpression.visit(this);
}
@Override
public String visit(final Expression.MatchAll expression) {
return "*";
}
@Override
public String visit(final Property expression) {
final String property = expression.property;
final String stringValue = expression.stringValue;
return property + "=" + stringValue;
}
}

63
pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/SyntaxException.java Normal file
View File

@@ -0,0 +1,63 @@
package org.lucares.pdb.keyword.db;
import org.antlr.v4.runtime.ParserRuleContext;
public class SyntaxException extends RuntimeException {
private static final long serialVersionUID = 1L;
private int lineStart;
private int startIndex;
private int lineStop;
private int stopIndex;
public SyntaxException(final ParserRuleContext context, final String message) {
super(message + ": " + generateMessage(context));
lineStart = context.getStart().getLine();
startIndex = context.getStart().getStartIndex();
lineStop = context.getStop().getLine();
stopIndex = context.getStop().getStopIndex();
}
private static String generateMessage(final ParserRuleContext context) {
final int lineStart = context.getStart().getLine();
final int startIndex = context.getStart().getStartIndex();
final int lineStop = context.getStop().getLine();
final int stopIndex = context.getStop().getStopIndex();
return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
}
public int getLineStart() {
return lineStart;
}
public void setLineStart(final int lineStart) {
this.lineStart = lineStart;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(final int startIndex) {
this.startIndex = startIndex;
}
public int getLineStop() {
return lineStop;
}
public void setLineStop(final int lineStop) {
this.lineStop = lineStop;
}
public int getStopIndex() {
return stopIndex;
}
public void setStopIndex(final int stopIndex) {
this.stopIndex = stopIndex;
}
}

99
pdb-keyword-db/src/test/java/org/lucares/pdb/keyword/db/KeywordsTest.java Normal file
View File

@@ -0,0 +1,99 @@
package org.lucares.pdb.keyword.db;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.testng.Assert;
import org.testng.annotations.Test;
@Test
public class KeywordsTest {
public void testKeywords() throws Exception {
final Keywords keywords = new Keywords();
final String file = "/tmp/abc";
final Map<String, String> tags = new HashMap<>();
tags.put("method", "m1");
tags.put("host", "h1");
tags.put("pod", "p1");
keywords.addFile(file, tags);
assertSearch(keywords, "method=m1", file);
assertSearch(keywords, "method=m1 AND host=h1", file);
assertSearch(keywords, "method=m1 OR pod=X", file);
query(keywords, "pod=pod3 and method=method124 or pod=pod3 and method=method125");
query(keywords, "!(pod=pod3)");
}
public void testKeywordsPerformance() throws Exception {
final Keywords keywords = new Keywords();
fill(keywords);
query(keywords, "host=host1");
query(keywords, "host=host1 and method=method123");
query(keywords, "host=host1 or host=host2");
query(keywords, "pod=pod3 and (method=method124 or method=method125)");
query(keywords, "pod=pod3 and method=method124 or pod=pod3 and method=method125");
query(keywords, "method=method124 and pod=pod3 or method=method125 and pod=pod3");
System.out.println(keywords.toString());
}
private void query(final Keywords keywords, final String query) {
final Expression expression = KeywordsLanguageParser.parse(query);
System.out.println(expression.visit(new PrintExpressionVisitor()));
for (int i = 0; i < 5; i++) {
keywords.search(query);
}
}
private void assertSearch(final Keywords keywords, final String query, final String... files) {
final Collection<String> actual = keywords.search(query);
Assert.assertEquals(actual, new HashSet<>(Arrays.asList(files)));
}
private void fill(final Keywords keywords) {
final List<String> pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "pod" + i).collect(Collectors.toList());
final List<String> hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList());
final List<String> versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList());
final List<String> methods = IntStream.rangeClosed(1, 200).mapToObj(i -> "method" + i)
.collect(Collectors.toList());
final List<String> types = Arrays.asList("app", "engine", "web", "batch");
for (final String pod : pods) {
for (final String host : hosts) {
for (final String version : versions) {
for (final String method : methods) {
for (final String type : types) {
final Map<String, String> tags = new HashMap<>();
final String file = Paths.get("/some/prefix", UUID.randomUUID().toString()).toString();
tags.put("pod", pod);
tags.put("host", host);
tags.put("method", method);
tags.put("version", version);
tags.put("type", type);
keywords.addFile(file, tags);
}
}
}
}
}
}
}

26
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -18,6 +18,7 @@ import org.lucares.recommind.logs.Plotter;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -62,7 +63,7 @@ public class PdbController implements HardcodedValues, CollectionUtils {
@RequestMapping(path = "/autocomplete", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // APPLICATION_FORM_URLENCODED_VALUE
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
@@ -94,11 +95,30 @@ public class PdbController implements HardcodedValues, CollectionUtils {
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
List<String> fields() {
List<String> fields(@RequestParam(name = "query") final String query) {
try {
final List<String> fields = db.getDb().getFields(query);
final List<String> fields = db.getDb().getFields();
return fields;
} catch (final Exception e) {
e.printStackTrace();
throw new InternalServerError(e);
}
}
@RequestMapping(path = "/fields/{fieldName}/values", //
method = RequestMethod.GET, //
consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
)
@ResponseBody
List<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) {
try {
final List<String> fields = db.getDb().getFieldsValues(query, fieldName);
return fields;

32
pdb-ui/src/main/resources/resources/css/design.css
View File

@@ -33,35 +33,30 @@ html, body {
#search-bar {
background-color: #aaa;
text-align: right;
padding-bottom:3px;
flex-grow: 0;
flex-shrink: 1;
flex-basis: auto;
}
#search-bar #search-query {
width:100%;
padding:2;
border:0;
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
#filter-bar {
}
#search-input-wrapper {
text-align: left;
display: flex;
padding: 5px;
}
#search-input-wrapper .autocomplete {
overflow-y: scroll;
background: white;
#add-filter {
float:right;
}
#search-bar #search-submit {
.template-filter-values {
}
#button-bar {
text-align: right;
}
#button-bar #search-submit {
margin-right:3px;
}
@@ -82,6 +77,7 @@ html, body {
line-height: 1.2em;
}
.center
{
display: flex;

26
pdb-ui/src/main/resources/resources/index.html
View File

@@ -3,12 +3,11 @@
<head>
<script type="text/javascript" src="js/jquery-3.1.1.min.js"></script>
<script type="text/javascript" src="js/search.js"></script>
<script type="text/javascript" src="js/autocomplete.js"></script>
<script type="text/javascript" src="js/mustache.min.js"></script>
<link rel="stylesheet" type="text/css" href="css/typography.css">
<link rel="stylesheet" type="text/css" href="css/design.css">
<link rel="stylesheet" type="text/css" href="css/icons.css">
<link rel="stylesheet" type="text/css" href="css/loading.css">
<link rel="stylesheet" type="text/css" href="css/autocomplete.min.css">
</head>
<body>
<div id="content">
@@ -16,16 +15,29 @@
<div id="logo">LOGO</div>
</div>
<div id="search-bar">
<div id="search-input-wrapper">
<input id="search-input" data-autocomplete="autocomplete"
data-autocomplete-empty-message="nothing found" />
<div id="filter-bar">
<button id="add-filter">add</button>
</div>
<div id="search-group-by">
<div id="button-bar">
<button id="search-submit"><i class="fa fa-search"> Search</i></button>
</div>
<button id="search-submit"><i class="fa fa-search"> Search</i></button>
</div>
<div id="result-view">
</div>
</div>
<div style="display:none">
<div id="template-filter">
<div class="filter">
<select class="template-filter-field" name="filter-field" id="filter-field">
{{#fields}}
<option name="{{name}}">{{name}}</option>
{{/fields}}
</select>
<select class="template-filter-values" name="filter-values" id="filter-values" multiple="multiple">
</select>
</div>
</div>
</div>
</body>
</html>

1
pdb-ui/src/main/resources/resources/js/mustache.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

111
pdb-ui/src/main/resources/resources/js/search.js
View File

@@ -3,48 +3,97 @@ $(document).ready(function(){
$('#search-submit').click(plot);
initGroupBy();
initFilter();
AutoComplete({
HttpMethod: "GET",
Delay: 300,
_QueryArg: function() {
var caretIndex = document.getElementById('search-input').selectionStart;
return 'caretIndex=' + caretIndex + '&query';
},
_Post: function(response) {
var result = [];
var responseObject = JSON.parse(response);
responseObject['proposals'].forEach(function(item, index){
var proposal = {};
proposal['Label'] = item.value;
proposal['Value'] = item.proposedQuery;
result.push(proposal);
});
console.log(JSON.stringify(result));
return result;
}
});
});
function initGroupBy()
{
var successCallback = function (response) {
function initFilter() {
response.forEach(function(item, index){
$('#search-group-by').append('<input type="radio" id="groupBy-'+index+'" name="groupBy" value="'+item+'" /><label for="groupBy-'+index+'">'+item+'</label>');
var successCallback = function (fields) {
var fieldsView = [{name: ""}];
fields.forEach(function(item, index){
fieldsView.push( {name: item} );
});
var template = $('#template-filter').html();
var view = {
fields: fieldsView
};
var rendered = Mustache.render(template, view);
$('#filter-bar').append(rendered);
$('#filter-values').hide();
$('#filter-field').change(function() {
if ($(this).val() == ""){
$('#filter-values').hide();
}else{
addFieldValues($(this).val());
}
});
$('#add-filter').click(function(){
var field = $('#filter-field').val();
var values = $('#filter-values').val();
if (field != ""
&& values != [] )
{
var query = "";
values.forEach(function (value, index) {
if (query.length > 0)
{
query += ",";
}
query += field+"="+value;
} );
$('#filter-bar').append(query);
}
});
};
getFields( successCallback);
}
function addFieldValues(field){
var params = {};
params['query'] = "";
var successCallback = function (values){
$('#result-view').text("SUCCESS: " + JSON.stringify(values));
values.forEach(function (item, index){
$('#filter-values').append($('<option>', {
value: item,
text : item
}));
});
$('#filter-values').show();
};
var errorCallback = function (){
$('#result-view').text("FAILED: " + JSON.stringify(e));
};
getJson("fields/"+field+"/values", params, successCallback);
}
function getFields( successCallback) {
var errorCallback = function (){
$('#result-view').text("FAILED: " + JSON.stringify(e));
};
getJson("fields", {}, successCallback, errorCallback)
var params = {};
params['query'] = "";
getJson("fields", params, successCallback, errorCallback)
}
function showLoadingIcon()

2
performanceDb/build.gradle
View File

@@ -1,7 +1,7 @@
dependencies {
compile project(':pdb-api')
compile 'org.lucares:ludb:1.0.20161223090600'
compile 'org.lucares:ludb:1.0.20170101101722'
compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5'
compile 'org.mapdb:mapdb:3.0.2'
}

9
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -16,6 +16,7 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldNotExistsException;
import org.lucares.ludb.H2DB;
import org.lucares.ludb.Proposal;
import org.lucares.pdb.api.Entry;
@@ -180,9 +181,13 @@ public class PerformanceDb implements AutoCloseable, CollectionUtils {
return db.proposeTagForQuery(query, caretIndex);
}
public List<String> getFields() {
final List<Field> fields = db.getAvailableFields("");
public List<String> getFields(final String query) {
final List<Field> fields = db.getAvailableFieldsForQuery(query);
return map(fields, Field::getName);
}
public List<String> getFieldsValues(final String query, final String fieldName) throws FieldNotExistsException {
return db.getAvailableValuesForField(query, fieldName);
}
}

3
performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java
View File

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
import java.io.File;
import org.lucares.ludb.Field;
import org.lucares.ludb.FieldNotExistsException;
import org.lucares.ludb.FieldType;
import org.lucares.ludb.H2DB;
@@ -12,7 +13,7 @@ class TagsUtils {
try {
db.setProperty(file, fieldName, value);
} catch (final FieldNotExistsException e) {
db.createField(fieldName, FieldType.STRING);
db.createField(new Field(fieldName, FieldType.STRING));
try {
db.setProperty(file, fieldName, value);
} catch (final FieldNotExistsException e1) {