group by multiple fields

Before we could only group by a single field. But it is acutally
very useful to group by multiple fields. For example to see the
graph for a small set of methods grouped by host and project.

pdb-api/src/main/java/org/lucares/pdb/api/Tags.java

@@ -1,7 +1,8 @@
 package org.lucares.pdb.api;
 
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -29,16 +30,25 @@ public class Tags {
 		return EMPTY;
 	}
 
+	public static Tags create(final String key, final String value) {
+		final Map<String, Tag> tags = new LinkedHashMap<>(1);
+		tags.put(key, new Tag(key, value));
+		return new Tags(tags);
+	}
+
 	public static Tags create(final String key1, final String value1, final String key2, final String value2) {
-		final Map<String, Tag> tags = new HashMap<>(2);
+		final Map<String, Tag> tags = new LinkedHashMap<>(2);
 		tags.put(key1, new Tag(key1, value1));
 		tags.put(key2, new Tag(key2, value2));
 		return new Tags(tags);
 	}
 
-	public static Tags create(final String key, final String value) {
+	public static Tags create(final String key1, final String value1, final String key2, final String value2,
-		final Map<String, Tag> tags = new HashMap<>(1);
+			final String key3, final String value3) {
-		tags.put(key, new Tag(key, value));
+		final Map<String, Tag> tags = new LinkedHashMap<>(3);
+		tags.put(key1, new Tag(key1, value1));
+		tags.put(key2, new Tag(key2, value2));
+		tags.put(key3, new Tag(key3, value3));
 		return new Tags(tags);
 	}
 
@@ -46,7 +56,7 @@ public class Tags {
 		Objects.requireNonNull(key, "key must not be null");
 		Objects.requireNonNull(value, "value must not be null");
 
-		final Map<String, Tag> newTags = new HashMap<>(tags);
+		final Map<String, Tag> newTags = new LinkedHashMap<>(tags);
 
 		newTags.put(key, new Tag(key, value));
 
@@ -82,7 +92,7 @@ public class Tags {
 
 	@Override
 	public String toString() {
-		return String.valueOf(tags);
+		return String.valueOf(tags.values());
 	}
 
 	@Override
@@ -142,4 +152,23 @@ public class Tags {
 		return s.substring(0, Math.min(maxLength, s.length()));
 	}
 
+	public Tags subset(final List<String> groupByFields) {
+
+		Tags result = new Tags();
+
+		for (final String field : groupByFields) {
+			final String value = getValue(field);
+
+			if (value != null) {
+				result = result.copyAdd(field, value);
+			}
+		}
+
+		return result;
+	}
+
+	public boolean isEmpty() {
+		return tags.isEmpty();
+	}
+
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PlotSettings.java

@@ -6,6 +6,7 @@ import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
+import java.util.List;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -19,7 +20,7 @@ public class PlotSettings {
 
 	private int width;
 
-	private String groupBy;
+	private List<String> groupBy;
 
 	private Limit limitBy;
 
@@ -53,11 +54,11 @@ public class PlotSettings {
 		this.width = width;
 	}
 
-	public String getGroupBy() {
+	public List<String> getGroupBy() {
 		return groupBy;
 	}
 
-	public void setGroupBy(final String groupBy) {
+	public void setGroupBy(final List<String> groupBy) {
 		this.groupBy = groupBy;
 	}
 

pdb-plotting/src/main/java/org/lucares/recommind/logs/Plotter.java

@@ -36,6 +36,8 @@ public class Plotter {
 	private static final Logger LOGGER = LoggerFactory.getLogger(Plotter.class);
 	private static final Logger METRICS_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.plotter");
 
+	private static final String DEFAULT_GROUP = "<none>";
+
 	private final PerformanceDb db;
 	private final Path tmpBaseDir;
 	private final Path outputDir;
@@ -68,7 +70,7 @@ public class Plotter {
 			final List<DataSeries> dataSeries = new ArrayList<>();
 
 			final String query = plotSettings.getQuery();
-			final String groupBy = plotSettings.getGroupBy();
+			final List<String> groupBy = plotSettings.getGroupBy();
 			final int height = plotSettings.getHeight();
 			final int width = plotSettings.getWidth();
 			final OffsetDateTime dateFrom = plotSettings.dateFrom();
@@ -165,12 +167,18 @@ public class Plotter {
 
 		final StringBuilder result = new StringBuilder();
 
-		assert tags.getKeys().size() <= 1;
+		if (tags.isEmpty()) {
-		tags.forEach((k, v) -> {
+			result.append(DEFAULT_GROUP);
-			result.append(v);
+		} else {
-		});
+			tags.forEach((k, v) -> {
+				if (result.length() > 0) {
+					result.append(" / ");
+				}
+				result.append(v);
+			});
+		}
 
-		result.append("(");
+		result.append(" (");
 		result.append(values);
 		result.append(")");
 

pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotRequest.java

@@ -1,5 +1,7 @@
 package org.lucares.pdbui.domain;
 
+import java.util.List;
+
 public class PlotRequest {
 	private String query;
 
@@ -7,7 +9,7 @@ public class PlotRequest {
 
 	private int width = 1000;
 
-	private String groupBy;
+	private List<String> groupBy;
 
 	private LimitBy limitBy = LimitBy.NO_LIMIT;
 
@@ -46,11 +48,11 @@ public class PlotRequest {
 		return query + ":" + height + "x" + width;
 	}
 
-	public String getGroupBy() {
+	public List<String> getGroupBy() {
 		return groupBy;
 	}
 
-	public void setGroupBy(final String groupBy) {
+	public void setGroupBy(final List<String> groupBy) {
 		this.groupBy = groupBy;
 	}
 

pdb-ui/src/main/resources/resources/js/search.js

@@ -52,18 +52,9 @@ function renderGroupBy()
 	
 	var success = function(response){
 		
-		$('#search-group-by').empty();
+		initSearchGroupBy('#search-group-by-1', response);
-		
+		initSearchGroupBy('#search-group-by-2', response);
-		var option = new Option("", "");
+		initSearchGroupBy('#search-group-by-3', response);
-		$('#search-group-by').append($(option));
-		
-		response.forEach(
-			(item, index) => {
-				var option = new Option(item, item);
-				$('#search-group-by').append($(option));
-			}
-		);
-		
 	};
 	var error = function(e) {
 		$('#result-view').text("FAILED: " + JSON.parse(e.responseText).message);
@@ -73,6 +64,20 @@ function renderGroupBy()
 	getJson("fields", request, success, error);
 }
 
+function initSearchGroupBy(selector, response)
+{
+	$(selector).empty();
+	var option = new Option("", "");
+	$(selector).append($(option));
+	
+	response.forEach(
+		(item, index) => {
+			var option = new Option(item, item);
+			$(selector).append($(option));
+		}
+	);
+}
+
 function showLoadingIcon()
 {
 	$('#result-view').html("<div class='center'><div class='uil-cube-css' style='-webkit-transform:scale(0.41)'><div /><div></div><div></div><div></div></div></div>");
@@ -86,7 +91,7 @@ function plot(event){
 	request['query'] = $('#search-input').val();
 	request['height'] = $('#result-view').height()-15;
 	request['width'] = $('#result-view').width()-15;
-	request['groupBy'] = $('#search-group-by').val();
+	request['groupBy'] = groupBy();
 	request['limitBy'] = $('#search-limit-by').val();
 	request['limit'] = parseInt($('#search-limit-value').val());
 	request['dateFrom'] = $('#search-date-from').val();
@@ -111,6 +116,20 @@ function plot(event){
 	
 }
 
+function groupBy()
+{
+	var result = [];
+	
+	for (var i = 1; i <= 3; i++)
+	{		
+		if ($('#search-group-by-'+i).val() != "")
+		{
+			result.push($('#search-group-by-'+i).val());
+		}
+	}
+	return result;
+}
+
 function postJson(url, requestData, successCallback, errorCallback) {
 
 	$.ajax({
@@ -135,3 +154,4 @@ function getJson(url, requestData, successCallback, errorCallback) {
 		.fail(errorCallback);
 }
 
+

pdb-ui/src/main/resources/templates/main.html

@@ -19,7 +19,10 @@
 			<input id="search-input" data-autocomplete="autocomplete"
 				data-autocomplete-empty-message="nothing found" />
 		</div>
-		<label for="search-group-by">Group By:</label> <select id="search-group-by"></select>
+		<label for="search-group-by-1">Group By:</label> 
+		<select id="search-group-by-1"></select>
+		<select id="search-group-by-2"></select>
+		<select id="search-group-by-3"></select>
 		
 		<label for="search-limit-by">Limit By:</label>
 		<select id="search-limit-by">

performanceDb/src/main/java/org/lucares/performance/db/Grouping.java

@@ -2,6 +2,7 @@ package org.lucares.performance.db;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -10,8 +11,8 @@ import org.lucares.pdb.api.Tags;
 
 public class Grouping {
 
-	public static final String NO_GROUPING = null;
+	public static final List<String> NO_GROUPING = Collections.emptyList();
-	public static final String DEFAULT_GROUP = "<none>";
+
 	private final List<Group> groups = new ArrayList<>();
 
 	private Grouping(final Group group) {
@@ -22,7 +23,7 @@ public class Grouping {
 		this.groups.addAll(groups);
 	}
 
-	public static Grouping groupBy(final List<PdbReader> pdbReaders, final String groupByField) {
+	public static Grouping groupBy(final List<PdbReader> pdbReaders, final List<String> groupByField) {
 
 		final Grouping result;
 		if (noGrouping(groupByField)) {
@@ -30,33 +31,29 @@ public class Grouping {
 
 			result = new Grouping(group);
 		} else {
-			final Map<String, Group> grouping = new HashMap<>();
+			final Map<Tags, Group> grouping = new HashMap<>();
 
 			for (final PdbReader pdbReader : pdbReaders) {
 				final Tags tags = pdbReader.getPdbFile().getTags();
-				final String value = tags.getValue(groupByField);
+				final Tags groupTags = tags.subset(groupByField);
 
-				final String groupName = value != null ? value : DEFAULT_GROUP;
+				addIfNotExists(grouping, groupTags);
-
+				grouping.get(groupTags).addReader(pdbReader);
-				addIfNotExists(grouping, groupByField, groupName);
-				grouping.get(groupName).addReader(pdbReader);
 			}
 			result = new Grouping(grouping.values());
 		}
 		return result;
 	}
 
-	private static boolean noGrouping(final String groupByField) {
+	private static boolean noGrouping(final List<String> groupByField) {
-		return groupByField == NO_GROUPING || groupByField.isEmpty();
+		return groupByField == null || groupByField.isEmpty();
 	}
 
-	private static void addIfNotExists(final Map<String, Group> grouping, final String groupByField,
+	private static void addIfNotExists(final Map<Tags, Group> grouping, final Tags groupTags) {
-			final String value) {
+		if (!grouping.containsKey(groupTags)) {
-		if (!grouping.containsKey(value)) {
-			final Tags tags = Tags.create(groupByField, value);
 			final List<PdbReader> readers = new ArrayList<>();
 
-			grouping.put(value, new Group(tags, readers));
+			grouping.put(groupTags, new Group(groupTags, readers));
 		}
 	}
 

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -152,7 +152,7 @@ public class PerformanceDb implements AutoCloseable, CollectionUtils {
 	 *            the tag to group by
 	 * @return {@link Result}
 	 */
-	public Result get(final String query, final String groupBy) {
+	public Result get(final String query, final List<String> groupBy) {
 
 		final List<PdbReader> pdbReaders = tagsToFile.getReaders(query);
 

performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java

@@ -166,7 +166,7 @@ public class PerformanceDbTest {
 		}
 	}
 
-	public void testGroupBy() throws Exception {
+	public void testGroupBySingleField() throws Exception {
 		try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
 			final OffsetDateTime from = DateUtils.getDate(2016, 1, 1, 00, 00, 00);
 			final OffsetDateTime to = DateUtils.getDate(2016, 1, 1, 23, 59, 50);
@@ -177,36 +177,77 @@ public class PerformanceDbTest {
 			final String key = "myKey";
 			final Tags tagsOne = Tags.create(key, "one", "commonKey", "commonValue");
 			final Tags tagsTwo = Tags.create(key, "two", "commonKey", "commonValue");
-			final Tags tagsThree = Tags.create(key, "three", "commonKey", "commonValue");
+			final Tags tagsThree = Tags.create("commonKey", "commonValue");
 			final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
 			final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwo, 2);
 			final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 3);
 
-			final Result result = db.get("commonKey=commonValue", key);
+			final Result result = db.get("commonKey=commonValue", Arrays.asList(key));
 
 			final List<GroupResult> groups = result.getGroups();
 
-			final List<String> actualGroup = new ArrayList<>();
 			for (final GroupResult groupResult : groups) {
-				final String groupedByValue = groupResult.getGroupedBy().getValue(key);
+				final Tags groupedBy = groupResult.getGroupedBy();
-				actualGroup.add(groupedByValue);
 
-				switch (groupedByValue) {
+				if (groupedBy.equals(Tags.create(key, "one"))) {
-				case "one":
 					Assert.assertEquals(groupResult.asList(), entriesOne);
-					break;
+				} else if (groupedBy.equals(Tags.create(key, "two"))) {
-				case "two":
-					Assert.assertEquals(groupResult.asList(), entriesTwo);
-					break;
-				case "three":
-					Assert.assertEquals(groupResult.asList(), entriesThree);
-					break;
 
-				default:
+					Assert.assertEquals(groupResult.asList(), entriesTwo);
-					break;
+				} else if (groupedBy.isEmpty()) {
+					Assert.assertEquals(groupResult.asList(), entriesThree);
+				} else {
+					Assert.fail("unexpected group: " + groupResult.getGroupedBy());
+				}
+			}
+		}
+	}
+
+	public void testGroupByMultipleFields() throws Exception {
+		try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+			final OffsetDateTime from = DateUtils.getDate(2016, 1, 1, 00, 00, 00);
+			final OffsetDateTime to = DateUtils.getDate(2016, 1, 1, 23, 59, 50);
+
+			final TimeRange timeRange = new TimeRange(from, to);
+			final long numberOfEntries = timeRange.duration().toHours();
+
+			final String key1 = "myKey1";
+			final String key2 = "myKey2";
+			final Tags tagsOne = Tags.create(key1, "one", key2, "aaa", "commonKey", "commonValue");
+			final Tags tagsTwoA = Tags.create(key1, "two", key2, "bbb", "commonKey", "commonValue");
+			final Tags tagsTwoB = Tags.create(key1, "two", key2, "bbb", "commonKey", "commonValue");
+			final Tags tagsThree = Tags.create(key1, "three", "commonKey", "commonValue");
+
+			final List<Entry> entriesOne = storeEntries(db, timeRange, numberOfEntries, tagsOne, 1);
+			final List<Entry> entriesTwo = storeEntries(db, timeRange, numberOfEntries, tagsTwoA, 2);
+			entriesTwo.addAll(storeEntries(db, timeRange, numberOfEntries, tagsTwoB, 3));
+			final List<Entry> entriesThree = storeEntries(db, timeRange, numberOfEntries, tagsThree, 4);
+
+			final Result result = db.get("commonKey=commonValue", Arrays.asList(key1, key2));
+
+			final List<GroupResult> groups = result.getGroups();
+
+			for (final GroupResult groupResult : groups) {
+				final Tags groupedBy = groupResult.getGroupedBy();
+
+				if (groupedBy.equals(Tags.create(key1, "one", key2, "aaa"))) {
+					Assert.assertEquals(groupResult.asList(), entriesOne);
+				} else if (groupedBy.equals(Tags.create(key1, "two", key2, "bbb"))) {
+					// there is no defined order of the entries.
+					// eventually we might return them in ascending order, but
+					// that is not yet implemented
+					final List<Entry> actualEntries = groupResult.asList();
+
+					entriesTwo.sort(EntryByDateComparator.INSTANCE);
+					actualEntries.sort(EntryByDateComparator.INSTANCE);
+
+					Assert.assertEquals(actualEntries, entriesTwo);
+				} else if (groupedBy.equals(Tags.create(key1, "three"))) {
+					Assert.assertEquals(groupResult.asList(), entriesThree);
+				} else {
+					Assert.fail("unexpected group: " + groupedBy);
 				}
 			}
-			Assert.assertEquals(actualGroup, Arrays.asList("one", "two", "three"));
 		}
 	}
 
@@ -225,8 +266,4 @@ public class PerformanceDbTest {
 			index++;
 		}
 	}
-
-	public void testBlockingIteratorInput() throws Exception {
-
-	}
 }