andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

group by multiple fields

Browse Source 
Before we could only group by a single field. But it is acutally
very useful to group by multiple fields. For example to see the
graph for a small set of methods grouped by host and project.
This commit is contained in:
Andreas Huber
2017-04-12 19:16:19 +02:00
parent 6cc6e679a4
commit 8baf05962f
9 changed files with 169 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
pdb-api/src/main/java/org/lucares/pdb/api/Tags.java
View File

@@ -1,7 +1,8 @@
package org.lucares.pdb.api;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -29,16 +30,25 @@ public class Tags {
return EMPTY;
}
public static Tags create(final String key, final String value) {
final Map<String, Tag> tags = new LinkedHashMap<>(1);
tags.put(key, new Tag(key, value));
return new Tags(tags);
}
public static Tags create(final String key1, final String value1, final String key2, final String value2) {
final Map<String, Tag> tags = new HashMap<>(2);
final Map<String, Tag> tags = new LinkedHashMap<>(2);
tags.put(key1, new Tag(key1, value1));
tags.put(key2, new Tag(key2, value2));
return new Tags(tags);
}
public static Tags create(final String key, final String value) {
final Map<String, Tag> tags = new HashMap<>(1);
tags.put(key, new Tag(key, value));
public static Tags create(final String key1, final String value1, final String key2, final String value2,
final String key3, final String value3) {
final Map<String, Tag> tags = new LinkedHashMap<>(3);
tags.put(key1, new Tag(key1, value1));
tags.put(key2, new Tag(key2, value2));
tags.put(key3, new Tag(key3, value3));
return new Tags(tags);
}
@@ -46,7 +56,7 @@ public class Tags {
Objects.requireNonNull(key, "key must not be null");
Objects.requireNonNull(value, "value must not be null");
final Map<String, Tag> newTags = new HashMap<>(tags);
final Map<String, Tag> newTags = new LinkedHashMap<>(tags);
newTags.put(key, new Tag(key, value));
@@ -82,7 +92,7 @@ public class Tags {
@Override
public String toString() {
return String.valueOf(tags);
return String.valueOf(tags.values());
}
@Override
@@ -142,4 +152,23 @@ public class Tags {
return s.substring(0, Math.min(maxLength, s.length()));
}
public Tags subset(final List<String> groupByFields) {
Tags result = new Tags();
for (final String field : groupByFields) {
final String value = getValue(field);
if (value != null) {
result = result.copyAdd(field, value);
}
}
return result;
}
public boolean isEmpty() {
return tags.isEmpty();
}
}

7
pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java
View File

@@ -6,6 +6,7 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
@@ -19,7 +20,7 @@ public class PlotSettings {
private int width;
private String groupBy;
private List<String> groupBy;
private Limit limitBy;
@@ -53,11 +54,11 @@ public class PlotSettings {
this.width = width;
}
public String getGroupBy() {
public List<String> getGroupBy() {
return groupBy;
}
public void setGroupBy(final String groupBy) {
public void setGroupBy(final List<String> groupBy) {
this.groupBy = groupBy;
}

14
pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java
View File

@@ -36,6 +36,8 @@ public class Plotter {
private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class);
private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter");
private static final String DEFAULT_GROUP = "<none>";
private final PerformanceDb db;
private final Path tmpBaseDir;
private final Path outputDir;
@@ -68,7 +70,7 @@ public class Plotter {
final List<DataSeries> dataSeries = new ArrayList<>();
final String query = plotSettings.getQuery();
final String groupBy = plotSettings.getGroupBy();
final List<String> groupBy = plotSettings.getGroupBy();
final int height = plotSettings.getHeight();
final int width = plotSettings.getWidth();
final OffsetDateTime dateFrom = plotSettings.dateFrom();
@@ -165,12 +167,18 @@ public class Plotter {
final StringBuilder result = new StringBuilder();
assert tags.getKeys().size() <= 1;
if (tags.isEmpty()) {
result.append(DEFAULT_GROUP);
} else {
tags.forEach((k, v) -> {
if (result.length() > 0) {
result.append(" / ");
}
result.append(v);
});
}
result.append("(");
result.append(" (");
result.append(values);
result.append(")");

8
pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java
View File

@@ -1,5 +1,7 @@
package org.lucares.pdbui.domain;
import java.util.List;
public class PlotRequest {
private String query;
@@ -7,7 +9,7 @@ public class PlotRequest {
private int width = 1000;
private String groupBy;
private List<String> groupBy;
private LimitBy limitBy = LimitBy.NO_LIMIT;
@@ -46,11 +48,11 @@ public class PlotRequest {
return query + ":" + height + "x" + width;
}
public String getGroupBy() {
public List<String> getGroupBy() {
return groupBy;
}
public void setGroupBy(final String groupBy) {
public void setGroupBy(final List<String> groupBy) {
this.groupBy = groupBy;
}

46
pdb-ui/src/main/resources/resources/js/search.js
View File

@@ -52,18 +52,9 @@ function renderGroupBy()
var success = function(response){
$('#search-group-by').empty();
var option = new Option("", "");
$('#search-group-by').append($(option));
response.forEach(
(item, index) => {
var option = new Option(item, item);
$('#search-group-by').append($(option));
}
);
initSearchGroupBy('#search-group-by-1', response);
initSearchGroupBy('#search-group-by-2', response);
initSearchGroupBy('#search-group-by-3', response);
};
var error = function(e) {
$('#result-view').text("FAILED: " + JSON.parse(e.responseText).message);
@@ -73,6 +64,20 @@ function renderGroupBy()
getJson("fields", request, success, error);
}
function initSearchGroupBy(selector, response)
{
$(selector).empty();
var option = new Option("", "");
$(selector).append($(option));
response.forEach(
(item, index) => {
var option = new Option(item, item);
$(selector).append($(option));
}
);
}
function showLoadingIcon()
{
$('#result-view').html("<div class='center'><div class='uil-cube-css' style='-webkit-transform:scale(0.41)'><div /><div></div><div></div><div></div></div></div>");
@@ -86,7 +91,7 @@ function plot(event){
request['query'] = $('#search-input').val();
request['height'] = $('#result-view').height()-15;
request['width'] = $('#result-view').width()-15;
request['groupBy'] = $('#search-group-by').val();
request['groupBy'] = groupBy();
request['limitBy'] = $('#search-limit-by').val();
request['limit'] = parseInt($('#search-limit-value').val());
request['dateFrom'] = $('#search-date-from').val();
@@ -111,6 +116,20 @@ function plot(event){
}
function groupBy()
{
var result = [];
for (var i = 1; i <= 3; i++)
{
if ($('#search-group-by-'+i).val() != "")
{
result.push($('#search-group-by-'+i).val());
}
}
return result;
}
function postJson(url, requestData, successCallback, errorCallback) {
$.ajax({
@@ -135,3 +154,4 @@ function getJson(url, requestData, successCallback, errorCallback) {
.fail(errorCallback);
}

5
pdb-ui/src/main/resources/templates/main.html
View File

@@ -19,7 +19,10 @@
<input id="search-input" data-autocomplete="autocomplete"
data-autocomplete-empty-message="nothing found" />
</div>
<label for="search-group-by">Group By:</label> <select id="search-group-by"></select>
<label for="search-group-by-1">Group By:</label>
<select id="search-group-by-1"></select>
<select id="search-group-by-2"></select>
<select id="search-group-by-3"></select>
<label for="search-limit-by">Limit By:</label>
<select id="search-limit-by">

29
performanceDb/src/main/java/org/lucares/performance/db/Grouping.java
View File

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -10,8 +11,8 @@ import org.lucares.pdb.api.Tags;
public class Grouping {
public static final String NO_GROUPING = null;
public static final String DEFAULT_GROUP = "<none>";
public static final List<String> NO_GROUPING = Collections.emptyList();
private final List<Group> groups = new ArrayList<>();
private Grouping(final Group group) {
@@ -22,7 +23,7 @@ public class Grouping {
this.groups.addAll(groups);
}
public static Grouping groupBy(final List<PdbReader> pdbReaders, final String groupByField) {
public static Grouping groupBy(final List<PdbReader> pdbReaders, final List<String> groupByField) {
final Grouping result;
if (noGrouping(groupByField)) {
@@ -30,33 +31,29 @@ public class Grouping {
result = new Grouping(group);
} else {
final Map<String, Group> grouping = new HashMap<>();
final Map<Tags, Group> grouping = new HashMap<>();
for (final PdbReader pdbReader : pdbReaders) {
final Tags tags = pdbReader.getPdbFile().getTags();
final String value = tags.getValue(groupByField);
final Tags groupTags = tags.subset(groupByField);
final String groupName = value != null ? value : DEFAULT_GROUP;
addIfNotExists(grouping, groupByField, groupName);
grouping.get(groupName).addReader(pdbReader);
addIfNotExists(grouping, groupTags);
grouping.get(groupTags).addReader(pdbReader);
}
result = new Grouping(grouping.values());
}
return result;
}
private static boolean noGrouping(final String groupByField) {
return groupByField == NO_GROUPING || groupByField.isEmpty();
private static boolean noGrouping(final List<String> groupByField) {
return groupByField == null || groupByField.isEmpty();
}
private static void addIfNotExists(final Map<String, Group> grouping, final String groupByField,
final String value) {
if (!grouping.containsKey(value)) {
final Tags tags = Tags.create(groupByField, value);
private static void addIfNotExists(final Map<Tags, Group> grouping, final Tags groupTags) {
if (!grouping.containsKey(groupTags)) {
final List<PdbReader> readers = new ArrayList<>();
grouping.put(value, new Group(tags, readers));
grouping.put(groupTags, new Group(groupTags, readers));
}
}

2
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -152,7 +152,7 @@ public class PerformanceDb implements AutoCloseable, CollectionUtils {
* the tag to group by
* @return {@link Result}
*/
public Result get(final String query, final String groupBy) {
public Result get(final String query, final List<String> groupBy) {
final List<PdbReader> pdbReaders = tagsToFile.getReaders(query);

81
performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java
View File

@@ -166,7 +166,7 @@ public class PerformanceDbTest {
}
}
public void testGroupBy() throws Exception {
public void testGroupBySingleField() throws Exception {
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final OffsetDateTime from = DateUtils.getDate(2016, 1, 1, 00, 00, 00);
final OffsetDateTime to = DateUtils.getDate(2016, 1, 1, 23, 59, 50);
@@ -177,36 +177,77 @@ public class PerformanceDbTest {
final String key = "myKey";
final Tags tagsOne = Tags.create(key, "one", "commonKey", "commonValue");
final Tags tagsTwo = Tags.create(key, "two", "commonKey", "commonValue");
final Tags tagsThree = Tags.create(key, "three", "commonKey", "commonValue");
final Tags tagsThree = Tags.create("commonKey", "commonValue");
final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwo, 2);
final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 3);
final Result result = db.get("commonKey=commonValue", key);
final Result result = db.get("commonKey=commonValue", Arrays.asList(key));
final List<GroupResult> groups = result.getGroups();
final List<String> actualGroup = new ArrayList<>();
for (final GroupResult groupResult : groups) {
final String groupedByValue = groupResult.getGroupedBy().getValue(key);
actualGroup.add(groupedByValue);
final Tags groupedBy = groupResult.getGroupedBy();
switch (groupedByValue) {
case "one":
if (groupedBy.equals(Tags.create(key, "one"))) {
Assert.assertEquals(groupResult.asList(), entriesOne);
break;
case "two":
Assert.assertEquals(groupResult.asList(), entriesTwo);
break;
case "three":
Assert.assertEquals(groupResult.asList(), entriesThree);
break;
} else if (groupedBy.equals(Tags.create(key, "two"))) {
default:
break;
Assert.assertEquals(groupResult.asList(), entriesTwo);
} else if (groupedBy.isEmpty()) {
Assert.assertEquals(groupResult.asList(), entriesThree);
} else {
Assert.fail("unexpected group: " + groupResult.getGroupedBy());
}
}
}
}
public void testGroupByMultipleFields() throws Exception {
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final OffsetDateTime from = DateUtils.getDate(2016, 1, 1, 00, 00, 00);
final OffsetDateTime to = DateUtils.getDate(2016, 1, 1, 23, 59, 50);
final TimeRange timeRange = new TimeRange(from, to);
final long numberOfEntries = timeRange.duration().toHours();
final String key1 = "myKey1";
final String key2 = "myKey2";
final Tags tagsOne = Tags.create(key1, "one", key2, "aaa", "commonKey", "commonValue");
final Tags tagsTwoA = Tags.create(key1, "two", key2, "bbb", "commonKey", "commonValue");
final Tags tagsTwoB = Tags.create(key1, "two", key2, "bbb", "commonKey", "commonValue");
final Tags tagsThree = Tags.create(key1, "three", "commonKey", "commonValue");
final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwoA, 2);
entriesTwo.addAll(storeEntries(db, timeRange, numberOfEntries, tagsTwoB, 3));
final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 4);
final Result result = db.get("commonKey=commonValue", Arrays.asList(key1, key2));
final List<GroupResult> groups = result.getGroups();
for (final GroupResult groupResult : groups) {
final Tags groupedBy = groupResult.getGroupedBy();
if (groupedBy.equals(Tags.create(key1, "one", key2, "aaa"))) {
Assert.assertEquals(groupResult.asList(), entriesOne);
} else if (groupedBy.equals(Tags.create(key1, "two", key2, "bbb"))) {
// there is no defined order of the entries.
// eventually we might return them in ascending order, but
// that is not yet implemented
final List<Entry> actualEntries = groupResult.asList();
entriesTwo.sort(EntryByDateComparator.INSTANCE);
actualEntries.sort(EntryByDateComparator.INSTANCE);
Assert.assertEquals(actualEntries, entriesTwo);
} else if (groupedBy.equals(Tags.create(key1, "three"))) {
Assert.assertEquals(groupResult.asList(), entriesThree);
} else {
Assert.fail("unexpected group: " + groupedBy);
}
}
Assert.assertEquals(actualGroup, Arrays.asList("one", "two", "three"));
}
}
@@ -225,8 +266,4 @@ public class PerformanceDbTest {
index++;
}
}
public void testBlockingIteratorInput() throws Exception {
}
}