From 4161cd7f98475d795473a04cbb592195c6a9cc54 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 27 Aug 2019 20:37:07 +0200 Subject: [PATCH] only field prefixes returned instead of full values When using autocomplete to return field values I missed, that autocomplete had the feature that cut values at dots. So instead of returning full field values only the prefix up to the first dot was returned. Fixed by making the cut-at-dot feature optional. --- .../pdb/datastore/lang/NewProposerParser.java | 22 +++- .../pdb/datastore/internal/DataStoreTest.java | 7 +- .../pdb/datastore/internal/ProposerTest.java | 117 ++++++++++++++---- .../lucares/pdb/api/QueryWithCaretMarker.java | 15 ++- .../java/org/lucares/pdbui/PdbController.java | 23 ++-- 5 files changed, 146 insertions(+), 38 deletions(-) diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java b/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java index 203ff0c..6b105a6 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/lang/NewProposerParser.java @@ -12,6 +12,7 @@ import org.apache.commons.lang3.StringUtils; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.QueryConstants; import org.lucares.pdb.api.QueryWithCaretMarker; +import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode; import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.datastore.internal.QueryCompletionIndex; import org.lucares.utils.CollectionUtils; @@ -51,10 +52,12 @@ public class NewProposerParser implements QueryConstants { proposals = foundProposals; } } + final List nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults()); + METRICS_LOGGER_PROPOSE.debug("compute proposals took {}ms for query '{}' ", (System.nanoTime() - start) / 1_000_000.0, query); - return proposals; + return nonEmptyProposals; } private List proposalsForNonValues(final QueryWithCaretMarker query) { @@ -167,10 +170,11 @@ public class NewProposerParser implements QueryConstants { final SortedSet candidateValues = normalizedExpression .visit(new FindValuesForQueryCompletion(query.getDateRange(), queryCompletionIndex)); - final SortedSet candidateValuesCutAtDots = cutAtDots(candidateValues, queryWithCaretMarker); + final SortedSet sortedAndPreparedCandidateValues = resultFilter(query.getResultMode(), + candidateValues, queryWithCaretMarker); // translate the candidate values to proposals - final List proposals = generateProposals(queryWithCaretMarker, candidateValuesCutAtDots); + final List proposals = generateProposals(queryWithCaretMarker, sortedAndPreparedCandidateValues); return proposals; } catch (final SyntaxException e) { @@ -180,6 +184,18 @@ public class NewProposerParser implements QueryConstants { } } + private SortedSet resultFilter(final ResultMode resultMode, final SortedSet candidateValues, + final String queryWithCaretMarker) { + switch (resultMode) { + case CUT_AT_DOT: + return cutAtDots(candidateValues, queryWithCaretMarker); + case FULL_VALUES: + return candidateValues; + default: + throw new IllegalArgumentException("Unexpected value: " + resultMode); + } + } + private SortedSet cutAtDots(final SortedSet candidateValues, final String queryWithCaretMarker) { final CandidateGrouper grouper = new CandidateGrouper(); return grouper.group(candidateValues, queryWithCaretMarker); diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java index ae8dde9..d333e63 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java @@ -26,6 +26,7 @@ import javax.swing.JTextField; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.Query; import org.lucares.pdb.api.QueryWithCaretMarker; +import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode; import org.lucares.pdb.api.Tags; import org.lucares.pdb.blockstorage.BSFile; import org.lucares.pdb.datastore.Doc; @@ -259,7 +260,8 @@ public class DataStoreTest { final String query = input.getText(); final int caretIndex = input.getCaretPosition(); - final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex); + final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, caretIndex, + ResultMode.CUT_AT_DOT); final List proposals = dataStore.propose(q); @@ -301,7 +303,8 @@ public class DataStoreTest { final List expectedProposedValues) { final String query = queryWithCaret.replace("|", ""); final int caretIndex = queryWithCaret.indexOf("|"); - final List proposals = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex)); + final List proposals = dataStore + .propose(new QueryWithCaretMarker(query, dateRange, caretIndex, ResultMode.CUT_AT_DOT)); System.out.println( "proposed values: " + proposals.stream().map(Proposal::getProposedTag).collect(Collectors.toList())); diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java index 587358f..9acf431 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java +++ b/data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java @@ -9,6 +9,7 @@ import java.util.List; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.QueryWithCaretMarker; +import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode; import org.lucares.pdb.api.Tags; import org.lucares.pdb.datastore.Proposal; import org.lucares.utils.CollectionUtils; @@ -51,51 +52,66 @@ public class ProposerTest { final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny"); final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim"); + final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web"); + final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source", + "service"); + final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web"); + final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service"); + dataStore.createNewFile(now, eagleTim); dataStore.createNewFile(now, eagleTimothy); dataStore.createNewFile(now, pigeonJennifer); dataStore.createNewFile(now, flamingoJennifer); dataStore.createNewFile(now, labradorJenny); dataStore.createNewFile(now, labradorTim); + + dataStore.createNewFile(now, methodA); + dataStore.createNewFile(now, methodB); + dataStore.createNewFile(now, methodC); + dataStore.createNewFile(now, methodD); } public void testEmptyQuery() throws Exception { - assertProposals("", 0, // + assertProposals("", ResultMode.FULL_VALUES, 0, // new Proposal("name", "name=*", true, "name=", 5), // new Proposal("bird", "bird=*", true, "bird=", 5), // - new Proposal("dog", "dog=*", true, "dog=", 4)// + new Proposal("dog", "dog=*", true, "dog=", 4), // + new Proposal("method", "method=*", true, "method=", 7), // + new Proposal("source", "source=*", true, "source=", 7)// ); - assertProposals(" ", 1, // + assertProposals(" ", ResultMode.FULL_VALUES, 1, // new Proposal("name", "name=*", true, "name=", 5), // new Proposal("bird", "bird=*", true, "bird=", 5), // - new Proposal("dog", "dog=*", true, "dog=", 4)// + new Proposal("dog", "dog=*", true, "dog=", 4), // + new Proposal("method", "method=*", true, "method=", 7), // + new Proposal("source", "source=*", true, "source=", 7)// ); } public void testPrefixOfKey() throws Exception { - assertProposals("bi", 2, // + assertProposals("bi", ResultMode.FULL_VALUES, 2, // new Proposal("bird", "bird=* ", true, "bird=", 5) // ); - assertProposals("bird", 4, // + assertProposals("bird", ResultMode.FULL_VALUES, 4, // new Proposal("bird", "bird=* ", true, "bird=", 5) // ); - assertProposals("bird=eagle and n", 16, // + assertProposals("bird=eagle and n", ResultMode.FULL_VALUES, 16, // new Proposal("name", "bird=eagle and name=* ", true, "bird=eagle and name=", 20) // ); } public void testPrefixOfValue() throws Exception { - assertProposals("name =Tim", 9, // + assertProposals("name =Tim", ResultMode.FULL_VALUES, 9, // new Proposal("Tim", "name =Tim", true, "name =Tim", 9), new Proposal("Timothy", "name =Timothy", true, "name =Timothy", 13)); - assertProposals("name =Je", 8, // + assertProposals("name =Je", ResultMode.FULL_VALUES, 8, // new Proposal("Jennifer", "name =Jennifer", true, "name =Jennifer", 14), // new Proposal("Jenny", "name =Jenny", true, "name =Jenny", 11) // ); - assertProposals("name =Tim,Je", 12, // + assertProposals("name =Tim,Je", ResultMode.FULL_VALUES, 12, // new Proposal("Jennifer", "name =Tim,Jennifer", true, "name =Tim,Jennifer", 18), // new Proposal("Jenny", "name =Tim,Jenny", true, "name =Tim,Jenny", 15) // ); @@ -105,18 +121,18 @@ public class ProposerTest { @Test(enabled = true) public void testInExpressions() throws Exception { - assertProposals("name = (Timothy,)", 16, // + assertProposals("name = (Timothy,)", ResultMode.FULL_VALUES, 16, // new Proposal("Jennifer", "name = (Timothy,Jennifer)", true, "name = (Timothy,Jennifer)", 24), // new Proposal("Jenny", "name = (Timothy,Jenny)", true, "name = (Timothy,Jenny)", 21), // new Proposal("Tim", "name = (Timothy,Tim)", true, "name = (Timothy,Tim)", 19), // new Proposal("Timothy", "name = (Timothy,Timothy)", true, "name = (Timothy,Timothy)", 23)// ); - assertProposals("name = (Timothy, J)", 18, // + assertProposals("name = (Timothy, J)", ResultMode.FULL_VALUES, 18, // new Proposal("Jennifer", "name = (Timothy, Jennifer)", true, "name = (Timothy, Jennifer)", 25), // new Proposal("Jenny", "name = (Timothy, Jenny)", true, "name = (Timothy, Jenny)", 22)); - assertProposals("name = (Tim)", 11, // + assertProposals("name = (Tim)", ResultMode.FULL_VALUES, 11, // new Proposal("Tim", "name = (Tim)", true, "name = (Tim)", 11), new Proposal("Timothy", "name = (Timothy)", true, "name = (Timothy)", 15)); @@ -125,46 +141,103 @@ public class ProposerTest { } public void testProposalOnEmptyValuePrefix() throws Exception { - assertProposals("name=", 5, // + assertProposals("name=", ResultMode.FULL_VALUES, 5, // new Proposal("Jennifer", "name=Jennifer", true, "name=Jennifer", 13), // new Proposal("Jenny", "name=Jenny", true, "name=Jenny", 10), // new Proposal("Tim", "name=Tim", true, "name=Tim", 8), // new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12) // ); + + assertProposals("method=", ResultMode.CUT_AT_DOT, 7, // + new Proposal("FooController.", "method=FooController.", true, "method=FooController.", 21), // + new Proposal("FooService.", "method=FooService.", true, "method=FooService.", 18), // + new Proposal("BarController.", "method=BarController.", true, "method=BarController.", 21), // + new Proposal("FooBarService.", "method=FooBarService.", true, "method=FooBarService.", 21) // + ); + assertProposals("method=", ResultMode.FULL_VALUES, 7, // + new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, + "method=FooController.doImportantStuff", 37), // + new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, + "method=FooService.doImportantStuff", 34), // + new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true, + "method=FooBarService.doOtherStuff", 33), // + new Proposal("BarController.doBoringStuff", "method=BarController.doBoringStuff", true, + "method=BarController.doBoringStuff", 34) // + ); + } + + public void testProposalOnValueSmartExpression() throws Exception { + assertProposals("method=Foo.", ResultMode.CUT_AT_DOT, 11, // + new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, + "method=FooController.doImportantStuff", 37), // + new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, + "method=FooService.doImportantStuff", 34), // + new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true, + "method=FooBarService.doOtherStuff", 33) // + ); + + assertProposals("method=Foo.*Stuf", ResultMode.CUT_AT_DOT, 16, // + new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, + "method=FooController.doImportantStuff", 37), // + new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, + "method=FooService.doImportantStuff", 34), // + new Proposal("FooBarService.doOtherStuff", "method=FooBarService.doOtherStuff", true, + "method=FooBarService.doOtherStuff", 33) // + ); + + // returns nothing, because GloblikePattern.globlikeToRegex() returns the + // following regex: ^[a-z]*Foo.*\.[a-z]*Stuf + // Maybe I will change that some day and allow upper case characters before + // "Stuff". + assertProposals("method=Foo.Stuf", ResultMode.CUT_AT_DOT, 15); + + assertProposals("method=Foo.Im", ResultMode.CUT_AT_DOT, 13, // + new Proposal("FooController.doImportantStuff", "method=FooController.doImportantStuff", true, + "method=FooController.doImportantStuff", 37), // + new Proposal("FooService.doImportantStuff", "method=FooService.doImportantStuff", true, + "method=FooService.doImportantStuff", 34) // + ); } public void testProposalOnEmptyKeyPrefix() throws Exception { - assertProposals("name=* and ", 11, // + assertProposals("name=* and ", ResultMode.FULL_VALUES, 11, // new Proposal("name", "name=* and name=* ", true, "name=* and name=", 16), // new Proposal("bird", "name=* and bird=* ", true, "name=* and bird=", 16), // - new Proposal("dog", "name=* and dog=* ", true, "name=* and dog=", 15) // + new Proposal("dog", "name=* and dog=* ", true, "name=* and dog=", 15), // + // TODO it is wrong to return those two, because there are no values with name + // and type|address, but I'll leave this for now, because this is a different + // issue + new Proposal("method", "name=* and method=* ", true, "name=* and method=", 18), // + new Proposal("source", "name=* and source=* ", true, "name=* and source=", 18)// ); } public void testProposalWithWildcards() throws Exception { - assertProposals("name=*im", 8, // + assertProposals("name=*im", ResultMode.FULL_VALUES, 8, // new Proposal("Tim", "name=Tim", true, "name=Tim", 8), // new Proposal("Timothy", "name=Timothy", true, "name=Timothy", 12)// ); + } public void testProposalWithAndExpression() throws Exception { - assertProposals("name=*im and bird=eagle", 8, // + assertProposals("name=*im and bird=eagle", ResultMode.FULL_VALUES, 8, // new Proposal("Tim", "name=Tim and bird=eagle", true, "name=Tim and bird=eagle", 8), // new Proposal("Timothy", "name=Timothy and bird=eagle", true, "name=Timothy and bird=eagle", 12)// ); - assertProposals("name=*im and bird=eagle,pigeon", 8, // + assertProposals("name=*im and bird=eagle,pigeon", ResultMode.FULL_VALUES, 8, // new Proposal("Tim", "name=Tim and bird=eagle,pigeon", true, "name=Tim and bird=eagle,pigeon", 8), // new Proposal("Timothy", "name=Timothy and bird=eagle,pigeon", true, "name=Timothy and bird=eagle,pigeon", 12)// ); } - private void assertProposals(final String query, final int caretIndex, final Proposal... expected) - throws InterruptedException { + private void assertProposals(final String query, final ResultMode resultMode, final int caretIndex, + final Proposal... expected) throws InterruptedException { - final List actual = dataStore.propose(new QueryWithCaretMarker(query, dateRange, caretIndex)); + final List actual = dataStore + .propose(new QueryWithCaretMarker(query, dateRange, caretIndex, resultMode)); final List expectedList = Arrays.asList(expected); Collections.sort(expectedList); diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java b/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java index 55df92f..31dcfae 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/QueryWithCaretMarker.java @@ -2,11 +2,18 @@ package org.lucares.pdb.api; public class QueryWithCaretMarker extends Query implements QueryConstants { - private final int caretIndex; + public enum ResultMode { + CUT_AT_DOT, FULL_VALUES + } - public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex) { + private final int caretIndex; + private final ResultMode resultMode; + + public QueryWithCaretMarker(final String query, final DateTimeRange dateRange, final int caretIndex, + final ResultMode resultMode) { super(query, dateRange); this.caretIndex = caretIndex; + this.resultMode = resultMode; } public String getQueryWithCaretMarker() { @@ -15,4 +22,8 @@ public class QueryWithCaretMarker extends Query implements QueryConstants { return queryWithCaretMarker.toString(); } + public ResultMode getResultMode() { + return resultMode; + } + } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 0b74896..29a08e6 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -18,6 +18,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.lucares.pdb.api.DateTimeRange; import org.lucares.pdb.api.QueryWithCaretMarker; +import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode; import org.lucares.pdb.datastore.Proposal; import org.lucares.pdb.plot.api.AxisScale; import org.lucares.pdb.plot.api.Limit; @@ -214,14 +215,14 @@ public class PdbController implements HardcodedValues, PropertyKeys { // TODO get date range from UI final DateTimeRange dateRange = DateTimeRange.max(); final int zeroBasedCaretIndex = caretIndex - 1; - final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, zeroBasedCaretIndex); + final QueryWithCaretMarker q = new QueryWithCaretMarker(query, dateRange, zeroBasedCaretIndex, + ResultMode.CUT_AT_DOT); final AutocompleteResponse result = new AutocompleteResponse(); final List proposals = db.autocomplete(q); - final List nonEmptyProposals = CollectionUtils.filter(proposals, p -> p.hasResults()); - final List autocompleteProposals = toAutocompleteProposals(nonEmptyProposals); + final List autocompleteProposals = toAutocompleteProposals(proposals); Collections.sort(autocompleteProposals, new AutocompleteProposalByValue()); result.setProposals(autocompleteProposals); @@ -252,14 +253,18 @@ public class PdbController implements HardcodedValues, PropertyKeys { SortedSet fields(@PathVariable(name = "fieldName") final String fieldName, @RequestParam(name = "query") final String query) { - final String q = String.format("(%s) and %s=", query, fieldName); - final int caretIndex = q.length() + 1; // the autocomplete methods needs a 1-based index + // TODO get date range from UI + final String q = query.isBlank()// + ? String.format("%s = ", fieldName)// + : String.format("(%s) and %s=", query, fieldName); + final int zeroBasedCaretIndex = q.length(); + final DateTimeRange dateRange = DateTimeRange.max(); + final QueryWithCaretMarker autocompleteQuery = new QueryWithCaretMarker(q, dateRange, zeroBasedCaretIndex, + ResultMode.FULL_VALUES); - final AutocompleteResponse autocompleteResponse = this.autocomplete(q, caretIndex); - final List proposals = autocompleteResponse.getProposals(); + final List result = db.autocomplete(autocompleteQuery); - final SortedSet fields = CollectionUtils.map(proposals, new TreeSet<>(), - AutocompleteProposal::getValue); + final SortedSet fields = CollectionUtils.map(result, new TreeSet<>(), Proposal::getProposedTag); return fields; }