andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

only field prefixes returned instead of full values

Browse Source 
When using autocomplete to return field values I
missed, that autocomplete had the feature that cut
values at dots. So instead of returning full field
values only the prefix up to the first dot was
returned.
Fixed by making the cut-at-dot feature optional.
This commit is contained in:
Andreas Huber
2019-08-27 20:37:07 +02:00
parent a174ec21ad
commit 4161cd7f98
5 changed files with 146 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java
View File

@@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.QueryConstants;
import org.lucares.pdb.api.QueryWithCaretMarker;
import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.internal.QueryCompletionIndex;
import org.lucares.utils.CollectionUtils;
@@ -51,10 +52,12 @@ public class NewProposerParser implements QueryConstants {
proposals = foundProposals;
}
}
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ",
(System.nanoTime() - start) / 1_000_000.0, query);
return proposals;
return nonEmptyProposals;
}
private List<Proposal> proposalsForNonValues(final QueryWithCaretMarker query) {
@@ -167,10 +170,11 @@ public class NewProposerParser implements QueryConstants {
final SortedSet<String> candidateValues = normalizedExpression
.visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex));
final SortedSet<String> candidateValuesCutAtDots = cutAtDots(candidateValues, queryWithCaretMarker);
final SortedSet<String> sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(),
candidateValues, queryWithCaretMarker);
// translate the candidate values to proposals
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, candidateValuesCutAtDots);
final List<Proposal> proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues);
return proposals;
} catch (final SyntaxException e) {
@@ -180,6 +184,18 @@ public class NewProposerParser implements QueryConstants {
}
}
private SortedSet<String> resultFilter(final ResultMode resultMode, final SortedSet<String> candidateValues,
final String queryWithCaretMarker) {
switch (resultMode) {
case CUT_AT_DOT:
return cutAtDots(candidateValues, queryWithCaretMarker);
case FULL_VALUES:
return candidateValues;
default:
throw new IllegalArgumentException("Unexpected value: " + resultMode);
}
}
private SortedSet<String> cutAtDots(final SortedSet<String> candidateValues, final String queryWithCaretMarker) {
final CandidateGrouper grouper = new CandidateGrouper();
return grouper.group(candidateValues, queryWithCaretMarker);

7
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -26,6 +26,7 @@ import javax.swing.JTextField;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.Query;
import org.lucares.pdb.api.QueryWithCaretMarker;
import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.Doc;
@@ -259,7 +260,8 @@ public class DataStoreTest {
final String query = input.getText();
final int caretIndex = input.getCaretPosition();
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex);
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex,
ResultMode.CUT_AT_DOT);
final List<Proposal> proposals = dataStore.propose(q);
@@ -301,7 +303,8 @@ public class DataStoreTest {
final List<String> expectedProposedValues) {
final String query = queryWithCaret.replace("|", "");
final int caretIndex = queryWithCaret.indexOf("|");
final List<Proposal> proposals = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex));
final List<Proposal> proposals = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT));
System.out.println(
"proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList()));

117
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -9,6 +9,7 @@ import java.util.List;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.QueryWithCaretMarker;
import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.utils.CollectionUtils;
@@ -51,51 +52,66 @@ public class ProposerTest {
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);
dataStore.createNewFile(now, pigeonJennifer);
dataStore.createNewFile(now, flamingoJennifer);
dataStore.createNewFile(now, labradorJenny);
dataStore.createNewFile(now, labradorTim);
dataStore.createNewFile(now, methodA);
dataStore.createNewFile(now, methodB);
dataStore.createNewFile(now, methodC);
dataStore.createNewFile(now, methodD);
}
public void testEmptyQuery() throws Exception {
assertProposals("", 0, //
assertProposals("", ResultMode.FULL_VALUES, 0, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4)//
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
assertProposals(" ", 1, //
assertProposals(" ", ResultMode.FULL_VALUES, 1, //
new Proposal("name", "name=*", true, "name=", 5), //
new Proposal("bird", "bird=*", true, "bird=", 5), //
new Proposal("dog", "dog=*", true, "dog=", 4)//
new Proposal("dog", "dog=*", true, "dog=", 4), //
new Proposal("method", "method=*", true, "method=", 7), //
new Proposal("source", "source=*", true, "source=", 7)//
);
}
public void testPrefixOfKey() throws Exception {
assertProposals("bi", 2, //
assertProposals("bi", ResultMode.FULL_VALUES, 2, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird", 4, //
assertProposals("bird", ResultMode.FULL_VALUES, 4, //
new Proposal("bird", "bird=* ", true, "bird=", 5) //
);
assertProposals("bird=eagle and n", 16, //
assertProposals("bird=eagle and n", ResultMode.FULL_VALUES, 16, //
new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) //
);
}
public void testPrefixOfValue() throws Exception {
assertProposals("name =Tim", 9, //
assertProposals("name =Tim", ResultMode.FULL_VALUES, 9, //
new Proposal("Tim", "name =Tim", true, "name =Tim", 9),
new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13));
assertProposals("name =Je", 8, //
assertProposals("name =Je", ResultMode.FULL_VALUES, 8, //
new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), //
new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) //
);
assertProposals("name =Tim,Je", 12, //
assertProposals("name =Tim,Je", ResultMode.FULL_VALUES, 12, //
new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), //
new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) //
);
@@ -105,18 +121,18 @@ public class ProposerTest {
@Test(enabled = true)
public void testInExpressions() throws Exception {
assertProposals("name = (Timothy,)", 16, //
assertProposals("name = (Timothy,)", ResultMode.FULL_VALUES, 16, //
new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), //
new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), //
new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), //
new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)//
);
assertProposals("name = (Timothy, J)", 18, //
assertProposals("name = (Timothy, J)", ResultMode.FULL_VALUES, 18, //
new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), //
new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22));
assertProposals("name = (Tim)", 11, //
assertProposals("name = (Tim)", ResultMode.FULL_VALUES, 11, //
new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11),
new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15));
@@ -125,46 +141,103 @@ public class ProposerTest {
}
public void testProposalOnEmptyValuePrefix() throws Exception {
assertProposals("name=", 5, //
assertProposals("name=", ResultMode.FULL_VALUES, 5, //
new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), //
new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) //
);
assertProposals("method=", ResultMode.CUT_AT_DOT, 7, //
new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), //
new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), //
new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), //
new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) //
);
assertProposals("method=", ResultMode.FULL_VALUES, 7, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33), //
new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true,
"method=BarController.doBoringStuff", 34) //
);
}
public void testProposalOnValueSmartExpression() throws Exception {
assertProposals("method=Foo.", ResultMode.CUT_AT_DOT, 11, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
assertProposals("method=Foo.*Stuf", ResultMode.CUT_AT_DOT, 16, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34), //
new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true,
"method=FooBarService.doOtherStuff", 33) //
);
// returns nothing, because GloblikePattern.globlikeToRegex() returns the
// following regex: ^[a-z]*Foo.*\.[a-z]*Stuf
// Maybe I will change that some day and allow upper case characters before
// "Stuff".
assertProposals("method=Foo.Stuf", ResultMode.CUT_AT_DOT, 15);
assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, //
new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true,
"method=FooController.doImportantStuff", 37), //
new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true,
"method=FooService.doImportantStuff", 34) //
);
}
public void testProposalOnEmptyKeyPrefix() throws Exception {
assertProposals("name=* and ", 11, //
assertProposals("name=* and ", ResultMode.FULL_VALUES, 11, //
new Proposal("name", "name=* and name=* ", true, "name=* and name=", 16), //
new Proposal("bird", "name=* and bird=* ", true, "name=* and bird=", 16), //
new Proposal("dog", "name=* and dog=* ", true, "name=* and dog=", 15) //
new Proposal("dog", "name=* and dog=* ", true, "name=* and dog=", 15), //
// TODO it is wrong to return those two, because there are no values with name
// and type|address, but I'll leave this for now, because this is a different
// issue
new Proposal("method", "name=* and method=* ", true, "name=* and method=", 18), //
new Proposal("source", "name=* and source=* ", true, "name=* and source=", 18)//
);
}
public void testProposalWithWildcards() throws Exception {
assertProposals("name=*im", 8, //
assertProposals("name=*im", ResultMode.FULL_VALUES, 8, //
new Proposal("Tim", "name=Tim", true, "name=Tim", 8), //
new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12)//
);
}
public void testProposalWithAndExpression() throws Exception {
assertProposals("name=*im and bird=eagle", 8, //
assertProposals("name=*im and bird=eagle", ResultMode.FULL_VALUES, 8, //
new Proposal("Tim", "name=Tim and bird=eagle", true, "name=Tim and bird=eagle", 8), //
new Proposal("Timothy", "name=Timothy and bird=eagle", true, "name=Timothy and bird=eagle", 12)//
);
assertProposals("name=*im and bird=eagle,pigeon", 8, //
assertProposals("name=*im and bird=eagle,pigeon", ResultMode.FULL_VALUES, 8, //
new Proposal("Tim", "name=Tim and bird=eagle,pigeon", true, "name=Tim and bird=eagle,pigeon", 8), //
new Proposal("Timothy", "name=Timothy and bird=eagle,pigeon", true,
"name=Timothy and bird=eagle,pigeon", 12)//
);
}
private void assertProposals(final String query, final int caretIndex, final Proposal... expected)
throws InterruptedException {
private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex,
final Proposal... expected) throws InterruptedException {
final List<Proposal> actual = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex));
final List<Proposal> actual = dataStore
.propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode));
final List<Proposal> expectedList = Arrays.asList(expected);
Collections.sort(expectedList);

15
pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java
View File

@@ -2,11 +2,18 @@ package org.lucares.pdb.api;
public class QueryWithCaretMarker extends Query implements QueryConstants {
private final int caretIndex;
public enum ResultMode {
CUT_AT_DOT, FULL_VALUES
}
public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex) {
private final int caretIndex;
private final ResultMode resultMode;
public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex,
final ResultMode resultMode) {
super(query, dateRange);
this.caretIndex = caretIndex;
this.resultMode = resultMode;
}
public String getQueryWithCaretMarker() {
@@ -15,4 +22,8 @@ public class QueryWithCaretMarker extends Query implements QueryConstants {
return queryWithCaretMarker.toString();
}
public ResultMode getResultMode() {
return resultMode;
}
}

23
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -18,6 +18,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.StringUtils;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.QueryWithCaretMarker;
import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.Limit;
@@ -214,14 +215,14 @@ public class PdbController implements HardcodedValues, PropertyKeys {
// TODO get date range from UI
final DateTimeRange dateRange = DateTimeRange.max();
final int zeroBasedCaretIndex = caretIndex - 1;
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, zeroBasedCaretIndex);
final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, zeroBasedCaretIndex,
ResultMode.CUT_AT_DOT);
final AutocompleteResponse result = new AutocompleteResponse();
final List<Proposal> proposals = db.autocomplete(q);
final List<Proposal> nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults());
final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(nonEmptyProposals);
final List<AutocompleteProposal> autocompleteProposals = toAutocompleteProposals(proposals);
Collections.sort(autocompleteProposals, new AutocompleteProposalByValue());
result.setProposals(autocompleteProposals);
@@ -252,14 +253,18 @@ public class PdbController implements HardcodedValues, PropertyKeys {
SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) {
final String q = String.format("(%s) and %s=", query, fieldName);
final int caretIndex = q.length() + 1; // the autocomplete methods needs a 1-based index
// TODO get date range from UI
final String q = query.isBlank()//
? String.format("%s = ", fieldName)//
: String.format("(%s) and %s=", query, fieldName);
final int zeroBasedCaretIndex = q.length();
final DateTimeRange dateRange = DateTimeRange.max();
final QueryWithCaretMarker autocompleteQuery = new QueryWithCaretMarker(q, dateRange, zeroBasedCaretIndex,
ResultMode.FULL_VALUES);
final AutocompleteResponse autocompleteResponse = this.autocomplete(q, caretIndex);
final List<AutocompleteProposal> proposals = autocompleteResponse.getProposals();
final List<Proposal> result = db.autocomplete(autocompleteQuery);
final SortedSet<String> fields = CollectionUtils.map(proposals, new TreeSet<>(),
AutocompleteProposal::getValue);
final SortedSet<String> fields = CollectionUtils.map(result, new TreeSet<>(), Proposal::getProposedTag);
return fields;
}