test for keywords db performance

pdb-keyword-db/.gitignore

@@ -0,0 +1,6 @@
+/bin/
+/build/
+/.settings/
+/.classpath
+/.project
+/test-output/

pdb-keyword-db/build.gradle

@@ -0,0 +1,17 @@
+
+apply plugin: 'antlr'
+
+dependencies {
+	compile 'org.apache.commons:commons-collections4:4.1'
+	runtime "org.antlr:antlr4:4.5.3"
+	antlr "org.antlr:antlr4:4.5.3" // use ANTLR version 4
+}
+
+sourceSets {
+   generated{
+        java.srcDir "build/generated-src/antlr/main"
+    }
+}
+compileJava{
+    source    += sourceSets.generated.java
+}

pdb-keyword-db/src/main/antlr/org/lucares/pdb/keyword/db/KeywordsLang.g4

@@ -0,0 +1,62 @@
+grammar KeywordsLang;
+
+@header {
+package org.lucares.pdb.keyword.db;
+}
+
+start : expression EOF ;
+
+expression 
+	: prop=identifier eq=equal value=identifier       #propertyExpression
+    | LPAREN expression RPAREN                       #parenExpression
+	| NOT expression                                 #notExpression
+    | left=expression op=and right=expression  #andExpression
+	| left=expression op=or right=expression #orExpression
+ ;
+  
+identifier 
+	: IDENTIFIER #identifierExpression
+ ;
+ 
+and : AND ;
+or : OR ;
+
+equal : EQUAL ;
+
+AND        : [aA][nN][dD] ;
+OR         : [oO][rR] ;
+NOT        : '!';
+EQUAL      : '=' ;
+LPAREN     : '(' ;
+RPAREN     : ')' ;
+WS         : [ \r\t\u000C\n]+ -> skip;
+
+IDENTIFIER
+    :   JavaLetter JavaLetterOrDigit*
+    ;
+
+fragment
+JavaLetter
+    :   [a-zA-Z$_] // these are the "java letters" below 0x7F
+    |   [\u002a] // asterisk, used for wildcards
+    |   // covers all characters above 0x7F which are not a surrogate
+        ~[\u0000-\u007F\uD800-\uDBFF]
+        {Character.isJavaIdentifierStart(_input.LA(-1))}?
+    |   // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
+        [\uD800-\uDBFF] [\uDC00-\uDFFF]
+        {Character.isJavaIdentifierStart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
+    ;
+
+fragment
+JavaLetterOrDigit
+    :   [a-zA-Z0-9$_] // these are the "java letters or digits" below 0x7F
+    |   [\u002a] // asterisk, used for wildcards
+    | '.'
+    | '/'
+    |   // covers all characters above 0x7F which are not a surrogate
+        ~[\u0000-\u007F\uD800-\uDBFF]
+        {Character.isJavaIdentifierPart(_input.LA(-1))}?
+    |   // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
+        [\uD800-\uDBFF] [\uDC00-\uDFFF]
+        {Character.isJavaIdentifierPart(Character.toCodePoint((char)_input.LA(-2), (char)_input.LA(-1)))}?
+    ;

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/Expression.java

@@ -0,0 +1,447 @@
+package org.lucares.pdb.keyword.db;
+
+abstract public class Expression {
+
+	public <T> T visit(final ExpressionVisitor<T> visitor) {
+		throw new UnsupportedOperationException();
+	}
+
+	abstract static class TemporaryExpression extends Expression {
+
+		abstract Expression toExpression(Expression left, Expression right);
+	}
+
+	static class OrTemporary extends TemporaryExpression {
+
+		@Override
+		Expression toExpression(final Expression left, final Expression right) {
+			return new Or(left, right);
+		}
+
+		@Override
+		public String toString() {
+			return "OrTemporary";
+		}
+	}
+
+	static class AndTemporary extends TemporaryExpression {
+		@Override
+		Expression toExpression(final Expression left, final Expression right) {
+			return new And(left, right);
+		}
+
+		@Override
+		public String toString() {
+			return "AndTemporary";
+		}
+	}
+
+	public static Or or(final Expression left, final Expression right) {
+		return new Or(left, right);
+	}
+
+	public static Or or(final String left, final String right) {
+		return new Or(term(left), term(right));
+	}
+
+	public static And and(final Expression left, final Expression right) {
+		return new And(left, right);
+	}
+
+	public static And and(final String left, final String right) {
+		return new And(term(left), term(right));
+	}
+
+	public static Terminal term(final String value) {
+		return new Terminal(value);
+	}
+
+	public static MatchAll matchAll() {
+		return MatchAll.INSTANCE;
+	}
+
+	public static Not not(final Expression expression) {
+		return new Not(expression);
+	}
+
+	public static Not not(final String expression) {
+		return new Not(term(expression));
+	}
+
+	public static Property property(final String property, final String value) {
+		return new Property(property, value);
+	}
+
+	public static Parentheses parentheses(final Expression expression) {
+		return new Parentheses(expression);
+	}
+
+	static class Not extends Expression {
+		private final Expression expression;
+
+		Not(final Expression expression) {
+			this.expression = expression;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		Expression getExpression() {
+			return expression;
+		}
+
+		@Override
+		public String toString() {
+			return "!" + expression;
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((expression == null) ? 0 : expression.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			final Not other = (Not) obj;
+			if (expression == null) {
+				if (other.expression != null) {
+					return false;
+				}
+			} else if (!expression.equals(other.expression)) {
+				return false;
+			}
+			return true;
+		}
+
+	}
+
+	static class Or extends Expression {
+		private final Expression left;
+		private final Expression right;
+
+		Or(final Expression left, final Expression right) {
+			this.left = left;
+			this.right = right;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		Expression getLeft() {
+			return left;
+		}
+
+		Expression getRight() {
+			return right;
+		}
+
+		@Override
+		public String toString() {
+
+			return " (" + left + " or " + right + ") ";
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((left == null) ? 0 : left.hashCode());
+			result = prime * result + ((right == null) ? 0 : right.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			final Or other = (Or) obj;
+			if (left == null) {
+				if (other.left != null) {
+					return false;
+				}
+			} else if (!left.equals(other.left)) {
+				return false;
+			}
+			if (right == null) {
+				if (other.right != null) {
+					return false;
+				}
+			} else if (!right.equals(other.right)) {
+				return false;
+			}
+			return true;
+		}
+
+	}
+
+	static class And extends Expression {
+		private final Expression left;
+		private final Expression right;
+
+		And(final Expression left, final Expression right) {
+			this.left = left;
+			this.right = right;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		Expression getLeft() {
+			return left;
+		}
+
+		Expression getRight() {
+			return right;
+		}
+
+		@Override
+		public String toString() {
+
+			return " (" + left + " and " + right + ") ";
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((left == null) ? 0 : left.hashCode());
+			result = prime * result + ((right == null) ? 0 : right.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			final And other = (And) obj;
+			if (left == null) {
+				if (other.left != null) {
+					return false;
+				}
+			} else if (!left.equals(other.left)) {
+				return false;
+			}
+			if (right == null) {
+				if (other.right != null) {
+					return false;
+				}
+			} else if (!right.equals(other.right)) {
+				return false;
+			}
+			return true;
+		}
+
+	}
+
+	static class MatchAll extends Expression {
+
+		public static final MatchAll INSTANCE = new MatchAll();
+
+		private MatchAll() {
+			//
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		@Override
+		public String toString() {
+
+			return "1=1";
+		}
+	}
+
+	static class Terminal extends Expression {
+		private final String value;
+
+		Terminal(final String value) {
+			this.value = value;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		@Override
+		public String toString() {
+
+			return value;
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((value == null) ? 0 : value.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			final Terminal other = (Terminal) obj;
+			if (value == null) {
+				if (other.value != null) {
+					return false;
+				}
+			} else if (!value.equals(other.value)) {
+				return false;
+			}
+			return true;
+		}
+
+		public String getValue() {
+			return value;
+		}
+	}
+
+	static class Property extends Expression {
+		final String property;
+		final String stringValue;
+
+		public Property(final String property, final String value) {
+			this.property = property;
+			this.stringValue = value;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		@Override
+		public String toString() {
+
+			return " " + property + " = " + stringValue + " ";
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((property == null) ? 0 : property.hashCode());
+			result = prime * result + ((stringValue == null) ? 0 : stringValue.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj)
+				return true;
+			if (obj == null)
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			final Property other = (Property) obj;
+			if (property == null) {
+				if (other.property != null)
+					return false;
+			} else if (!property.equals(other.property))
+				return false;
+			if (stringValue == null) {
+				if (other.stringValue != null)
+					return false;
+			} else if (!stringValue.equals(other.stringValue))
+				return false;
+			return true;
+		}
+
+	}
+
+	static class Parentheses extends Expression {
+		private final Expression expression;
+
+		Parentheses(final Expression expression) {
+			this.expression = expression;
+		}
+
+		@Override
+		public <T> T visit(final ExpressionVisitor<T> visitor) {
+			return visitor.visit(this);
+		}
+
+		public Expression getExpression() {
+			return expression;
+		}
+
+		@Override
+		public String toString() {
+
+			return " [ " + expression + " ] ";
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((expression == null) ? 0 : expression.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(final Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			final Parentheses other = (Parentheses) obj;
+			if (expression == null) {
+				if (other.expression != null) {
+					return false;
+				}
+			} else if (!expression.equals(other.expression)) {
+				return false;
+			}
+			return true;
+		}
+	}
+
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/ExpressionToFilesVisitor.java

@@ -0,0 +1,92 @@
+package org.lucares.pdb.keyword.db;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.lucares.pdb.keyword.db.Expression.And;
+import org.lucares.pdb.keyword.db.Expression.Not;
+import org.lucares.pdb.keyword.db.Expression.Or;
+import org.lucares.pdb.keyword.db.Expression.Property;
+
+public class ExpressionToFilesVisitor extends ExpressionVisitor<Set<String>> {
+
+	private static final Set<String> EMPTY = new TreeSet<>();
+
+	private final Map<String, Map<String, SortedSet<String>>> tagToFiles;
+	private final Map<String, Map<String, String>> fileToTags;
+
+	public ExpressionToFilesVisitor(final Map<String, Map<String, SortedSet<String>>> tagToFiles,
+			final Map<String, Map<String, String>> fileToTags) {
+		this.tagToFiles = tagToFiles;
+		this.fileToTags = fileToTags;
+	}
+
+	@Override
+	public Set<String> visit(final And expression) {
+
+		final Expression left = expression.getLeft();
+		final Expression right = expression.getRight();
+
+		final Set<String> leftFiles = left.visit(this);
+		final Set<String> rightFiles = right.visit(this);
+
+		final Set<String> result = new HashSet<>(leftFiles);
+		result.retainAll(rightFiles);
+		return result;
+
+	}
+
+	@Override
+	public Set<String> visit(final Or expression) {
+		final Expression left = expression.getLeft();
+		final Expression right = expression.getRight();
+
+		final Set<String> leftFiles = left.visit(this);
+		final Set<String> rightFiles = right.visit(this);
+
+		final Set<String> result = new HashSet<>(leftFiles);
+		result.addAll(rightFiles);
+		return result;
+	}
+
+	@Override
+	public Set<String> visit(final Not expression) {
+
+		final Expression negatedExpression = expression.getExpression();
+
+		final Set<String> files = negatedExpression.visit(this);
+		final Set<String> result = new HashSet<>(fileToTags.keySet());
+		result.removeAll(files);
+		return result;
+	}
+
+	@Override
+	public Set<String> visit(final Expression.MatchAll expression) {
+
+		return fileToTags.keySet();
+	}
+
+	@Override
+	public Set<String> visit(final Property expression) {
+
+		final Set<String> result;
+		final String property = expression.property;
+		final String stringValue = expression.stringValue;
+		final Map<String, SortedSet<String>> values = tagToFiles.get(property);
+		if (values != null) {
+			final SortedSet<String> files = values.get(stringValue);
+			if (files != null) {
+				result = files;
+			} else {
+				result = EMPTY;
+			}
+		} else {
+			result = EMPTY;
+		}
+
+		return result;
+	}
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/ExpressionVisitor.java

@@ -0,0 +1,31 @@
+package org.lucares.pdb.keyword.db;
+
+public abstract class ExpressionVisitor<T> {
+	public T visit(final Expression.And expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.Or expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.Not expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.Property expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.Terminal expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.MatchAll expression) {
+		throw new UnsupportedOperationException();
+	}
+
+	public T visit(final Expression.Parentheses parentheses) {
+		throw new UnsupportedOperationException();
+	}
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/Keywords.java

@@ -0,0 +1,57 @@
+package org.lucares.pdb.keyword.db;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class Keywords {
+
+	final Map<String, Map<String, String>> fileToTags = new HashMap<>();
+
+	final Map<String, Map<String, SortedSet<String>>> tagToFiles = new HashMap<>();
+
+	public void addFile(final String file, final Map<String, String> tags) {
+
+		fileToTags.put(file, tags);
+
+		for (final Entry<String, String> e : tags.entrySet()) {
+
+			final String field = e.getKey();
+			final String value = e.getValue();
+
+			tagToFiles.putIfAbsent(field, new HashMap<>());
+			final Map<String, SortedSet<String>> fieldToFiles = tagToFiles.get(field);
+			fieldToFiles.putIfAbsent(value, new TreeSet<>());
+			final SortedSet<String> files = fieldToFiles.get(value);
+			files.add(file);
+		}
+	}
+
+	public Collection<String> search(final String query) {
+		final long start = System.nanoTime();
+		final Expression expression = KeywordsLanguageParser.parse(query);
+		long duration = System.nanoTime() - start;
+		final String parsing = "parsing: " + duration / 1_000_000.0 + "ms";
+		// System.out.println(expression.visit(new PrintExpressionVisitor()));
+
+		final ExpressionToFilesVisitor visitor = new ExpressionToFilesVisitor(tagToFiles, fileToTags);
+
+		final long start2 = System.nanoTime();
+		final Set<String> result = expression.visit(visitor);
+		long duration2 = System.nanoTime() - start2;
+		System.out.println(
+				parsing + "; searching: " + duration2 / 1_000_000.0 + "ms; found=" + result.size());
+
+		return result;
+	}
+
+	@Override
+	public String toString() {
+		return "Keywords [files=" + fileToTags.size() + ", fields=" + tagToFiles.size() + "]";
+	}
+
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/KeywordsLangLanguage.java

@@ -0,0 +1,122 @@
+package org.lucares.pdb.keyword.db;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Stack;
+
+import org.antlr.v4.gui.TreeViewer;
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.DiagnosticErrorListener;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeListener;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+import org.lucares.pdb.keyword.db.Expression.And;
+import org.lucares.pdb.keyword.db.Expression.Not;
+import org.lucares.pdb.keyword.db.Expression.Or;
+import org.lucares.pdb.keyword.db.Expression.Terminal;
+import org.lucares.pdb.keyword.db.KeywordsLangParser.AndExpressionContext;
+import org.lucares.pdb.keyword.db.KeywordsLangParser.IdentifierExpressionContext;
+import org.lucares.pdb.keyword.db.KeywordsLangParser.NotExpressionContext;
+import org.lucares.pdb.keyword.db.KeywordsLangParser.OrExpressionContext;
+import org.lucares.pdb.keyword.db.KeywordsLangParser.PropertyExpressionContext;
+
+public class KeywordsLangLanguage {
+
+	public Expression parse(final String input) {
+		// define the input
+		final ANTLRInputStream in = new ANTLRInputStream(input);
+
+		// create lexer and parser
+		final KeywordsLangLexer lexer = new KeywordsLangLexer(in);
+		lexer.addErrorListener(new DiagnosticErrorListener());
+
+		final CommonTokenStream tokens = new CommonTokenStream(lexer);
+		final KeywordsLangParser parser = new KeywordsLangParser(tokens);
+
+		final Stack<Expression> stack = new Stack<>();
+
+		// define a listener that is called for every terminals and
+		// non-terminals
+		final ParseTreeListener listener = new KeywordsLangBaseListener() {
+
+			@Override
+			public void exitIdentifierExpression(final IdentifierExpressionContext ctx) {
+				// System.out.println("push identifier " + ctx.getText());
+
+				if (ctx.getText().length() > 255) {
+					throw new SyntaxException(ctx, "token too long");
+				}
+
+				stack.push(new Terminal(ctx.getText()));
+			}
+
+			@Override
+			public void exitPropertyExpression(final PropertyExpressionContext ctx) {
+				// System.out.println("property expression");
+
+				final Expression value = stack.pop();
+				final Terminal property = (Terminal) stack.pop();
+
+				if (value instanceof Terminal) {
+					stack.push(new Expression.Property(property.getValue(), ((Terminal) value).getValue()));
+
+				} else {
+					throw new UnsupportedOperationException();
+				}
+
+			}
+
+			@Override
+			public void exitNotExpression(final NotExpressionContext ctx) {
+
+				final Expression expression = stack.pop();
+
+				final Expression notExpression = new Not(expression);
+				stack.push(notExpression);
+			}
+
+			@Override
+			public void exitAndExpression(final AndExpressionContext ctx) {
+
+				final Expression right = stack.pop();
+				final Expression left = stack.pop();
+
+				stack.push(new And(left, right));
+			}
+
+			@Override
+			public void exitOrExpression(final OrExpressionContext ctx) {
+				final Expression right = stack.pop();
+				final Expression left = stack.pop();
+
+				stack.push(new Or(left, right));
+			}
+		};
+
+		// Specify our entry point
+		final ParseTree parseTree = parser.start();
+
+		// Walk it and attach our listener
+		final ParseTreeWalker walker = new ParseTreeWalker();
+		walker.walk(listener, parseTree);
+
+		if (stack.size() != 1) {
+			throw new RuntimeException("stack should have exactly one element  " + stack);
+		}
+
+		return stack.pop();
+	}
+
+	public static void main(final String[] args) {
+		final org.antlr.v4.runtime.CharStream stream = new ANTLRInputStream("prop=value");
+		final KeywordsLangLexer lexer = new KeywordsLangLexer(stream);
+		final CommonTokenStream tokens = new CommonTokenStream(lexer);
+		final KeywordsLangParser parser = new KeywordsLangParser(tokens);
+		final ParseTree tree = parser.start();
+		final List<String> ruleNames = Arrays.asList(KeywordsLangParser.ruleNames);
+		final TreeViewer view = new TreeViewer(ruleNames, tree);
+		view.open();
+	}
+
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/KeywordsLanguageParser.java

@@ -0,0 +1,15 @@
+package org.lucares.pdb.keyword.db;
+
+public class KeywordsLanguageParser {
+	public static Expression parse(final String query) {
+
+		final Expression result;
+		if (query == null || query.length() == 0) {
+			result = Expression.matchAll();
+		} else {
+			final KeywordsLangLanguage lang = new KeywordsLangLanguage();
+			result = lang.parse(query);
+		}
+		return result;
+	}
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/PrintExpressionVisitor.java

@@ -0,0 +1,53 @@
+package org.lucares.pdb.keyword.db;
+
+import org.lucares.pdb.keyword.db.Expression.And;
+import org.lucares.pdb.keyword.db.Expression.Not;
+import org.lucares.pdb.keyword.db.Expression.Or;
+import org.lucares.pdb.keyword.db.Expression.Property;
+
+public class PrintExpressionVisitor extends ExpressionVisitor<String> {
+
+	public PrintExpressionVisitor() {
+	}
+
+	@Override
+	public String visit(final And expression) {
+
+		final Expression left = expression.getLeft();
+		final Expression right = expression.getRight();
+
+		return "(" + left.visit(this) + " and " + right.visit(this) + ")";
+
+	}
+
+	@Override
+	public String visit(final Or expression) {
+		final Expression left = expression.getLeft();
+		final Expression right = expression.getRight();
+
+		return "(" + left.visit(this) + " or " + right.visit(this) + ")";
+	}
+
+	@Override
+	public String visit(final Not expression) {
+
+		final Expression negatedExpression = expression.getExpression();
+
+		return "!" + negatedExpression.visit(this);
+	}
+
+	@Override
+	public String visit(final Expression.MatchAll expression) {
+
+		return "*";
+	}
+
+	@Override
+	public String visit(final Property expression) {
+
+		final String property = expression.property;
+		final String stringValue = expression.stringValue;
+
+		return property + "=" + stringValue;
+	}
+}

pdb-keyword-db/src/main/java/org/lucares/pdb/keyword/db/SyntaxException.java

@@ -0,0 +1,63 @@
+package org.lucares.pdb.keyword.db;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+
+public class SyntaxException extends RuntimeException {
+
+	private static final long serialVersionUID = 1L;
+	private int lineStart;
+	private int startIndex;
+	private int lineStop;
+	private int stopIndex;
+
+	public SyntaxException(final ParserRuleContext context, final String message) {
+		super(message + ": " + generateMessage(context));
+
+		lineStart = context.getStart().getLine();
+		startIndex = context.getStart().getStartIndex();
+		lineStop = context.getStop().getLine();
+		stopIndex = context.getStop().getStopIndex();
+	}
+
+	private static String generateMessage(final ParserRuleContext context) {
+
+		final int lineStart = context.getStart().getLine();
+		final int startIndex = context.getStart().getStartIndex();
+		final int lineStop = context.getStop().getLine();
+		final int stopIndex = context.getStop().getStopIndex();
+
+		return String.format("line=%d, start=%d, to line=%d stop=%d", lineStart, startIndex, lineStop, stopIndex);
+	}
+
+	public int getLineStart() {
+		return lineStart;
+	}
+
+	public void setLineStart(final int lineStart) {
+		this.lineStart = lineStart;
+	}
+
+	public int getStartIndex() {
+		return startIndex;
+	}
+
+	public void setStartIndex(final int startIndex) {
+		this.startIndex = startIndex;
+	}
+
+	public int getLineStop() {
+		return lineStop;
+	}
+
+	public void setLineStop(final int lineStop) {
+		this.lineStop = lineStop;
+	}
+
+	public int getStopIndex() {
+		return stopIndex;
+	}
+
+	public void setStopIndex(final int stopIndex) {
+		this.stopIndex = stopIndex;
+	}
+}

pdb-keyword-db/src/test/java/org/lucares/pdb/keyword/db/KeywordsTest.java

@@ -0,0 +1,99 @@
+package org.lucares.pdb.keyword.db;
+
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+@Test
+public class KeywordsTest {
+	public void testKeywords() throws Exception {
+
+		final Keywords keywords = new Keywords();
+
+		final String file = "/tmp/abc";
+		final Map<String, String> tags = new HashMap<>();
+
+		tags.put("method", "m1");
+		tags.put("host", "h1");
+		tags.put("pod", "p1");
+
+		keywords.addFile(file, tags);
+
+		assertSearch(keywords, "method=m1", file);
+		assertSearch(keywords, "method=m1 AND host=h1", file);
+		assertSearch(keywords, "method=m1 OR pod=X", file);
+		query(keywords, "pod=pod3 and method=method124 or pod=pod3 and method=method125");
+		query(keywords, "!(pod=pod3)");
+	}
+
+	public void testKeywordsPerformance() throws Exception {
+
+		final Keywords keywords = new Keywords();
+
+		fill(keywords);
+
+		query(keywords, "host=host1");
+		query(keywords, "host=host1 and method=method123");
+		query(keywords, "host=host1 or host=host2");
+		query(keywords, "pod=pod3 and (method=method124 or method=method125)");
+		query(keywords, "pod=pod3 and method=method124 or pod=pod3 and method=method125");
+		query(keywords, "method=method124 and pod=pod3 or method=method125 and pod=pod3");
+
+		System.out.println(keywords.toString());
+	}
+
+	private void query(final Keywords keywords, final String query) {
+		final Expression expression = KeywordsLanguageParser.parse(query);
+		System.out.println(expression.visit(new PrintExpressionVisitor()));
+		for (int i = 0; i < 5; i++) {
+
+			keywords.search(query);
+		}
+	}
+
+	private void assertSearch(final Keywords keywords, final String query, final String... files) {
+		final Collection<String> actual = keywords.search(query);
+		Assert.assertEquals(actual, new HashSet<>(Arrays.asList(files)));
+	}
+
+	private void fill(final Keywords keywords) {
+
+		final List<String> pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "pod" + i).collect(Collectors.toList());
+		final List<String> hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList());
+		final List<String> versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList());
+		final List<String> methods = IntStream.rangeClosed(1, 200).mapToObj(i -> "method" + i)
+				.collect(Collectors.toList());
+		final List<String> types = Arrays.asList("app", "engine", "web", "batch");
+
+		for (final String pod : pods) {
+			for (final String host : hosts) {
+				for (final String version : versions) {
+					for (final String method : methods) {
+						for (final String type : types) {
+							final Map<String, String> tags = new HashMap<>();
+
+							final String file = Paths.get("/some/prefix", UUID.randomUUID().toString()).toString();
+							tags.put("pod", pod);
+							tags.put("host", host);
+							tags.put("method", method);
+							tags.put("version", version);
+							tags.put("type", type);
+
+							keywords.addFile(file, tags);
+						}
+					}
+				}
+			}
+		}
+	}
+}

pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java

@@ -18,6 +18,7 @@ import org.lucares.recommind.logs.Plotter;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -62,7 +63,7 @@ public class PdbController implements HardcodedValues, CollectionUtils {
 
 	@RequestMapping(path = "/autocomplete", //
 			method = RequestMethod.GET, //
-			consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE, //
+			consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, // APPLICATION_FORM_URLENCODED_VALUE
 			produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
 	)
 	@ResponseBody
@@ -94,11 +95,30 @@ public class PdbController implements HardcodedValues, CollectionUtils {
 			produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
 	)
 	@ResponseBody
-	List<String> fields() {
+	List<String> fields(@RequestParam(name = "query") final String query) {
 
 		try {
+			final List<String> fields = db.getDb().getFields(query);
 
-			final List<String> fields = db.getDb().getFields();
+			return fields;
+
+		} catch (final Exception e) {
+			e.printStackTrace();
+			throw new InternalServerError(e);
+		}
+	}
+
+	@RequestMapping(path = "/fields/{fieldName}/values", //
+			method = RequestMethod.GET, //
+			consumes = MediaType.APPLICATION_JSON_UTF8_VALUE, //
+			produces = MediaType.APPLICATION_JSON_UTF8_VALUE //
+	)
+	@ResponseBody
+	List<String> fields(@PathVariable(name = "fieldName") final String fieldName,
+			@RequestParam(name = "query") final String query) {
+
+		try {
+			final List<String> fields = db.getDb().getFieldsValues(query, fieldName);
 
 			return fields;
 

pdb-ui/src/main/resources/resources/css/design.css

@@ -33,35 +33,30 @@ html, body {
 
 #search-bar {
 	background-color: #aaa;
-	text-align: right;
 	padding-bottom:3px;
 	flex-grow: 0;
 	flex-shrink: 1;
 	flex-basis: auto;
 }
 
-#search-bar #search-query {
+#filter-bar {
-	width:100%;
+	
-	padding:2;
-	border:0;
-    -webkit-box-sizing: border-box;
-    -moz-box-sizing: border-box;
-    box-sizing: border-box;
 }
 
-#search-input-wrapper {
+#add-filter {
-	text-align: left;
+	float:right;
-	display: flex;
-	padding: 5px;
-}
-
-#search-input-wrapper .autocomplete {
-	overflow-y: scroll;
-	background: white;
 }
 
 
-#search-bar #search-submit {
+.template-filter-values {
+	
+}
+
+#button-bar {
+	text-align: right;
+}
+
+#button-bar #search-submit {
 	margin-right:3px;
 }
 
@@ -82,6 +77,7 @@ html, body {
     line-height: 1.2em;
 }
 
+
 .center
 {
     display: flex;

pdb-ui/src/main/resources/resources/index.html

@@ -3,12 +3,11 @@
 <head>
 <script type="text/javascript" src="js/jquery-3.1.1.min.js"></script>
 <script type="text/javascript" src="js/search.js"></script>
-<script type="text/javascript" src="js/autocomplete.js"></script>
+<script type="text/javascript" src="js/mustache.min.js"></script>
 <link rel="stylesheet" type="text/css" href="css/typography.css">
 <link rel="stylesheet" type="text/css" href="css/design.css">
 <link rel="stylesheet" type="text/css" href="css/icons.css">
 <link rel="stylesheet" type="text/css" href="css/loading.css">
-<link rel="stylesheet" type="text/css" href="css/autocomplete.min.css">
 </head>
 <body>
 	<div id="content">
@@ -16,16 +15,29 @@
 			<div id="logo">LOGO</div>
 		</div>
 		<div id="search-bar">
-			<div id="search-input-wrapper">
+			<div id="filter-bar">
-				<input id="search-input" data-autocomplete="autocomplete"
+				<button id="add-filter">add</button>
-					data-autocomplete-empty-message="nothing found" />
 			</div>
-			<div id="search-group-by">
+			
+			<div id="button-bar">
+				<button id="search-submit"><i class="fa fa-search"> Search</i></button>
 			</div>
-			<button id="search-submit"><i class="fa fa-search"> Search</i></button>
 		</div>
 		<div id="result-view">
 		</div>
 	</div>
+	<div style="display:none">
+		<div id="template-filter">
+			<div class="filter">
+				<select class="template-filter-field" name="filter-field" id="filter-field">
+					{{#fields}}
+					<option name="{{name}}">{{name}}</option>
+					{{/fields}}
+				</select>
+				<select class="template-filter-values" name="filter-values" id="filter-values" multiple="multiple">
+				</select>
+			</div>
+		</div>
+	</div>
 </body>
 </html>

pdb-ui/src/main/resources/resources/js/search.js

@@ -3,48 +3,97 @@ $(document).ready(function(){
 
 	$('#search-submit').click(plot);
 	
-	initGroupBy();
+	initFilter();
+	
 	
-	AutoComplete({
-		HttpMethod: "GET",
-		Delay: 300,
-		_QueryArg: function() {
-			var caretIndex = document.getElementById('search-input').selectionStart;
-			return 'caretIndex=' + caretIndex + '&query';
-		},
-		_Post: function(response) {
-			var result = [];
-			var responseObject = JSON.parse(response);
-			responseObject['proposals'].forEach(function(item, index){
-				var proposal = {};
-				proposal['Label'] = item.value;
-				proposal['Value'] = item.proposedQuery;
-			
-				result.push(proposal);
-			});
-			
-			console.log(JSON.stringify(result));
-			return result;
-		}
-	});
 });
 
-function initGroupBy()
+
-{
+function initFilter() {
-	var successCallback = function (response) {
 	
-		response.forEach(function(item, index){
+	var successCallback = function (fields) {
-			$('#search-group-by').append('<input type="radio" id="groupBy-'+index+'" name="groupBy" value="'+item+'" /><label for="groupBy-'+index+'">'+item+'</label>');
+		
+  		var fieldsView = [{name: ""}];
+  		fields.forEach(function(item, index){
+  			fieldsView.push( {name: item} );
+  		}); 
+  
+		var template = $('#template-filter').html();
+		var view = {
+			fields: fieldsView
+		};
+		
+		var rendered = Mustache.render(template, view);
+		$('#filter-bar').append(rendered);
+		
+		$('#filter-values').hide();
+		
+		$('#filter-field').change(function() {
+			if ($(this).val() == ""){
+				$('#filter-values').hide();
+			}else{
+				addFieldValues($(this).val());
+			}
+		});
+		
+		$('#add-filter').click(function(){
+			var field = $('#filter-field').val();
+			var values = $('#filter-values').val();
+		
+			if (field != ""
+			  && values != [] )
+			{
+				var query = "";
+				values.forEach(function (value, index) {
+					if (query.length > 0)
+					{
+						query += ",";
+					}
+					query += field+"="+value;
+				} );
+				$('#filter-bar').append(query);
+			}
 		});
-	
 	};
 	
+	getFields( successCallback);
+}
+
+function addFieldValues(field){
+	
+	var params = {};
+	params['query'] = ""; 
+
+	var successCallback = function (values){
+		$('#result-view').text("SUCCESS: " + JSON.stringify(values));
+
+		values.forEach(function (item, index){
+			$('#filter-values').append($('<option>', { 
+				value: item,
+				text : item 
+			}));
+		});
+		
+		$('#filter-values').show();
+	};
+
+	var errorCallback = function (){
+		$('#result-view').text("FAILED: " + JSON.stringify(e));
+	};
+
+	getJson("fields/"+field+"/values", params, successCallback);
+}
+
+function getFields( successCallback) {
+	
 	var errorCallback = function (){
 		$('#result-view').text("FAILED: " + JSON.stringify(e));
 	};
 
-	getJson("fields", {}, successCallback, errorCallback)
+	var params = {};
-	
+	params['query'] = "";
+
+	getJson("fields", params, successCallback, errorCallback)
 }
 
 function showLoadingIcon()

performanceDb/build.gradle

@@ -1,7 +1,7 @@
 
 dependencies {
 	compile project(':pdb-api')
-	compile 'org.lucares:ludb:1.0.20161223090600'
+	compile 'org.lucares:ludb:1.0.20170101101722'
 	compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5'
 	compile 'org.mapdb:mapdb:3.0.2'
 }

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -16,6 +16,7 @@ import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 import org.lucares.ludb.Field;
+import org.lucares.ludb.FieldNotExistsException;
 import org.lucares.ludb.H2DB;
 import org.lucares.ludb.Proposal;
 import org.lucares.pdb.api.Entry;
@@ -180,9 +181,13 @@ public class PerformanceDb implements AutoCloseable, CollectionUtils {
 		return db.proposeTagForQuery(query, caretIndex);
 	}
 
-	public List<String> getFields() {
+	public List<String> getFields(final String query) {
-		final List<Field> fields = db.getAvailableFields("");
+		final List<Field> fields = db.getAvailableFieldsForQuery(query);
 
 		return map(fields, Field::getName);
 	}
+
+	public List<String> getFieldsValues(final String query, final String fieldName) throws FieldNotExistsException {
+		return db.getAvailableValuesForField(query, fieldName);
+	}
 }

performanceDb/src/main/java/org/lucares/performance/db/TagsUtils.java

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
 
 import java.io.File;
 
+import org.lucares.ludb.Field;
 import org.lucares.ludb.FieldNotExistsException;
 import org.lucares.ludb.FieldType;
 import org.lucares.ludb.H2DB;
@@ -12,7 +13,7 @@ class TagsUtils {
 		try {
 			db.setProperty(file, fieldName, value);
 		} catch (final FieldNotExistsException e) {
-			db.createField(fieldName, FieldType.STRING);
+			db.createField(new Field(fieldName, FieldType.STRING));
 			try {
 				db.setProperty(file, fieldName, value);
 			} catch (final FieldNotExistsException e1) {