add box plots

pdb-js/src/app/plot.service.ts

@@ -17,7 +17,8 @@ export class PlotService {
     this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration));
     this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount));
     this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count));
-    this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", true, DataType.Group, DataType.Count));
+    this.plotTypes.push(new PlotType("BAR", "Bar (number of requests)", "bar-chart", true, DataType.Group, DataType.Count));
+    this.plotTypes.push(new PlotType("BOX", "Box", "box-plot", true, DataType.Time, DataType.Duration));
    
     this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration));

pdb-js/src/app/visualization-page/visualization-page.component.html

@@ -31,7 +31,7 @@
       </mat-form-field>
       
       <pdb-limit-by #limitbycomponent></pdb-limit-by>
-      <div [hidden]="!selectedPlotTypesContains('BAR')">
+      <div [hidden]="!selectedPlotTypesContains(['BAR', 'BOX'])">
       <mat-form-field >
         <mat-label>Intervals (only bar chart):</mat-label>
         <mat-select [(value)]="intervalUnit">
@@ -46,7 +46,7 @@
         </mat-select>
       </mat-form-field>
       </div>
-      <div [hidden]="!selectedPlotTypesContains('BAR')">
+      <div [hidden]="!selectedPlotTypesContains(['BAR', 'BOX'])">
         <mat-checkbox [(ngModel)]="renderBarChartTickLabels">Show Tic Labels (bar chart)</mat-checkbox>
       </div>
      <pdb-y-axis-definition #y1AxisDefinitionComponent yIndex="1"></pdb-y-axis-definition>

pdb-js/src/app/visualization-page/visualization-page.component.ts

@@ -92,8 +92,8 @@ export class VisualizationPageComponent implements OnInit {
     this.y2AxisAvailable = axesTypes.y.length == 2;
   }
   
-  selectedPlotTypesContains(plotTypeId: string){
+  selectedPlotTypesContains(plotTypeIds: Array<string>){
-    return this.selectedPlotType.filter(pt => pt.id == plotTypeId).length > 0;
+    return this.selectedPlotType.filter(pt => plotTypeIds.includes(pt.id)).length > 0;
   }
    
   

pdb-js/src/assets/img/box-plot.svg

@@ -0,0 +1,25 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 32 32">
+<g transform="translate(0.5,0.5)">
+	<rect x="5" y="8" width="8" height="15" style="fill: none; stroke: black;stroke-width:2;" />
+	<line x1="6" y1="3" x2="12" y2="3" style="stroke:black;stroke-width:2;"/>
+	<line x1="9" y1="8" x2="9" y2="3" style="stroke:black;stroke-width:2;"/>
+	<line x1="5" y1="15" x2="13" y2="15" style="stroke:black;stroke-width:2;"/>
+	<line x1="9" y1="23" x2="9" y2="28" style="stroke:black;stroke-width:2;"/>
+	<line x1="6" y1="28" x2="12" y2="28" style="stroke:black;stroke-width:2;"/>
+
+	<rect x="18" y="6" width="8" height="13" style="fill: none; stroke: black; stroke-width:2;" />
+	<line x1="19" y1="2" x2="25" y2="2" style="stroke:black;stroke-width:2;"/>
+	<line x1="22" y1="6" x2="22" y2="2" style="stroke:black;stroke-width:2;"/>
+	<line x1="18" y1="13" x2="26" y2="13" style="stroke:black;stroke-width:2;"/>
+	<line x1="22" y1="19" x2="22" y2="26" style="stroke:black;stroke-width:2;"/>
+	<line x1="19" y1="26" x2="25" y2="26" style="stroke:black;stroke-width:2;"/>
+	
+	<path d="M1,0
+	         L1,30
+	         L32,30"
+	      style="stroke:black; stroke-width: 3px; fill:none;"/>
+</g>
+</svg>
+
+
+

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java

@@ -18,7 +18,9 @@ public enum Aggregate {
      */
     CUM_DISTRIBUTION("Cumulative Distribution"),
 
-    HISTOGRAM("Histogram");
+    HISTOGRAM("Histogram"),
+
+    BOX("Box");
 
     private final String axisLabel;
 

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BarChartAggregatorForIntervals.java

@@ -30,7 +30,7 @@ public class BarChartAggregatorForIntervals implements CustomAggregator, Indexed
     public BarChartAggregatorForIntervals(final PlotSettings settings) {
         this.settings = settings;
         this.interval = settings.getInterval().get();
-        buckets = interval.getBuckets();
+        buckets = interval.getBuckets(AtomicLong::new);
     }
 
     @Override

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BoxAggregator.java

@@ -0,0 +1,101 @@
+package org.lucares.pdb.plot.api;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.UUID;
+
+import org.lucares.recommind.logs.GnuplotAxis;
+
+public class BoxAggregator implements CustomAggregator, IndexedAggregator {
+
+    private static final double SPACE_BETWEEN_BARS = 0.6;
+    private Long index = null;
+    private Long numberOfDataSeries;
+
+    private final String dataName = "$data" + UUID.randomUUID().toString().replace("-", "");
+
+    private final Interval interval;
+
+    private final Map<Long, PercentilesAggregator> buckets = new HashMap<>();
+
+    public BoxAggregator(final PlotSettings settings) {
+
+        this.interval = settings.getInterval().get();
+    }
+
+    @Override
+    public void addValue(final long epochMilli, final long value) {
+
+        final long bucketId = interval.toBucketMiddleTime(epochMilli);
+        buckets.putIfAbsent(bucketId, new PercentilesAggregator());
+        buckets.get(bucketId).addValue(epochMilli, value);
+    }
+
+    @Override
+    public AggregatedData getAggregatedData() {
+        // not needed - usually this method is used to write the data to file, but bar
+        // charts use inline data
+        return null;
+    }
+
+    @Override
+    public Aggregate getType() {
+        return Aggregate.BOX;
+    }
+
+    @Override
+    public void setIndex(final long index, final long numberOfDataSeries) {
+        this.index = index;
+        this.numberOfDataSeries = numberOfDataSeries;
+    }
+
+    @Override
+    public long getIndex() throws IllegalStateException {
+        if (this.index == null) {
+            throw new IllegalStateException("index was not set");
+        }
+        return this.index;
+    }
+
+    @Override
+    public long getNumberOfDataSeries() throws IllegalStateException {
+        if (this.numberOfDataSeries == null) {
+            throw new IllegalStateException("index was not set");
+        }
+        return this.numberOfDataSeries;
+    }
+
+    public Object getDataName() {
+        return dataName;
+    }
+
+    public Interval getInterval() {
+        return interval;
+    }
+
+    public String asCsv(final boolean renderLabels) {
+        final StringBuilder csv = new StringBuilder();
+        int i = 0;
+        for (final long bucketId : buckets.keySet()) {
+            final Percentiles percentiles = buckets.get(bucketId).getPercentiles();
+            csv.append(String.format(Locale.US, "%d,%d,%d,%d,%d,%d", //
+                    bucketId / 1000, //
+                    percentiles.get("0.000"), //
+                    percentiles.get("25.000"), //
+                    percentiles.get("50.000"), //
+                    percentiles.get("75.000"), //
+                    percentiles.get("100.000")//
+            ));
+            csv.append("\n");
+            i++;
+        }
+        return csv.toString();
+    }
+
+    public String renderLabels(final GnuplotAxis xAxis) {
+        final StringBuilder result = new StringBuilder();
+        return result.toString();
+    }
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/BoxChartHandler.java

@@ -0,0 +1,116 @@
+package org.lucares.pdb.plot.api;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Optional;
+
+import org.lucares.recommind.logs.AxisSettings;
+import org.lucares.recommind.logs.AxisTime;
+import org.lucares.recommind.logs.DataSeries;
+import org.lucares.recommind.logs.GnuplotAxis;
+import org.lucares.recommind.logs.GnuplotLineType;
+import org.lucares.recommind.logs.GnuplotSettings;
+import org.lucares.recommind.logs.LineStyle;
+import org.lucares.recommind.logs.Type;
+
+public class BoxChartHandler extends AggregateHandler {
+
+    @Override
+    Type getAxisType(final GnuplotAxis axis) {
+        switch (axis) {
+        case X1:
+        case X2:
+            return Type.Time;
+        case Y1:
+        case Y2:
+            return Type.Duration;
+        default:
+            throw new IllegalArgumentException("Unexpected value: " + axis);
+        }
+    }
+
+    @Override
+    Aggregate getAggregateType() {
+        return Aggregate.BOX;
+    }
+
+    @Override
+    AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
+        final AxisSettings result = AxisTime.createXAxis(settings);
+        result.setAxis(getxAxis());
+        result.setShowGrid(getxAxis() == GnuplotAxis.X1);
+
+        return result;
+    }
+
+    @Override
+    String beforePlot(final CustomAggregator aggregator, final GnuplotSettings settings) {
+        final StringBuilder result = new StringBuilder();
+
+        final BoxAggregator boxAggregator = (BoxAggregator) aggregator;
+
+        appendfln(result, "%s <<EOD", boxAggregator.getDataName());
+        appendln(result, boxAggregator.asCsv(settings.isRenderLabels()));
+        appendln(result, "EOD");
+
+        if (settings.isRenderLabels() && settings.isRenderBarChartTickLabels()) {
+            appendfln(result, boxAggregator.renderLabels(getxAxis()));
+        }
+
+        return result.toString();
+    }
+
+    @Override
+    String addPlot(final CustomAggregator aggregator, final LineStyle lineStyle, final Optional<String> title) {
+        final BoxAggregator boxAggregator = (BoxAggregator) aggregator;
+
+        final String candlestick = formatln(
+                "'%s' using 1:3:2:6:5:(%.1f) %s axes %s with %s whiskerbars 0.5 fs empty %s linewidth 1, \\", //
+                boxAggregator.getDataName(), //
+                width(boxAggregator.getInterval().getIntervalTimeUnit()), //
+                gnuplotTitle(title), //
+                gnuplotXYAxis(), //
+                GnuplotLineType.BOX, //
+                lineStyle.asGnuplotLineStyle()//
+        );
+        final String median = formatln(
+                "'%s' using 1:4:4:4:4:(%.1f) axes %s with candlesticks notitle fs empty %s linewidth 2, \\", //
+                boxAggregator.getDataName(), //
+                width(boxAggregator.getInterval().getIntervalTimeUnit()), //
+                gnuplotXYAxis(), //
+                lineStyle.asGnuplotLineStyle());
+        return candlestick + median;
+    }
+
+    private double width(final IntervalTimeUnit intervalTimeUnit) {
+        switch (intervalTimeUnit) {
+        case SECOND:
+            return 1;
+        case MINUTE:
+            return 60;
+        case HOUR:
+            return 3600;
+        case DAY:
+            return 86400;
+        case WEEK:
+            return 24 * 3600;
+        case MONTH:
+            return 30 * 24 * 3600;
+        case YEAR:
+            return 365 * 24 * 3600;
+        default:
+            throw new IllegalArgumentException("Unexpected value: " + this);
+        }
+    }
+
+    @Override
+    CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
+            final long fromEpochMilli, final long toEpochMilli) {
+        if (plotSettings.getInterval().isPresent()) {
+            return new BoxAggregator(plotSettings);
+        } else {
+            return null;
+        }
+    }
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Interval.java

@@ -3,7 +3,7 @@ package org.lucares.pdb.plot.api;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Supplier;
 
 import org.lucares.pdb.api.DateTimeRange;
 import org.lucares.pdb.datastore.internal.LongToDateBucket;
@@ -51,6 +51,27 @@ public class Interval {
         return bucketer.toPartitionId(epochMilli);
     }
 
+    public long toBucketMiddleTime(final long epochMilli) {
+        switch (intervalTimeUnit) {
+        case SECOND:
+            return epochMilli - epochMilli % 1000 + 500;
+        case MINUTE:
+            return epochMilli - epochMilli % 60000 + 30000;
+        case HOUR:
+            return epochMilli - epochMilli % 3600000 + 1800000;
+        case DAY:
+            return epochMilli - epochMilli % 86400000 + 43200000;
+//        case WEEK:
+//            return "YYYY-ww"; // use week based year! Otherwise intervals over the year boundary will be wrong
+//        case MONTH:
+//            return "yyyy-MM";
+//        case YEAR:
+//            return "yyyy";
+        default:
+            throw new IllegalArgumentException("Unexpected value: " + intervalTimeUnit);
+        }
+    }
+
     public IntervalTimeUnit getIntervalTimeUnit() {
         return intervalTimeUnit;
     }
@@ -72,13 +93,13 @@ public class Interval {
         return null;
     }
 
-    public Map<String, AtomicLong> getBuckets() {
+    public <T> Map<String, T> getBuckets(final Supplier<T> initialValueSupplier) {
-        final Map<String, AtomicLong> result = new HashMap<>();
+        final Map<String, T> result = new HashMap<>();
         final List<String> bucketIds = bucketer.toPartitionIds(dateTimeRange.getStart(), dateTimeRange.getEnd(),
                 intervalTimeUnit.toChronoUnit());
 
         for (final String bucketId : bucketIds) {
-            result.put(bucketId, new AtomicLong(0));
+            result.put(bucketId, initialValueSupplier.get());
         }
 
         return result;

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/PercentilesAggregator.java

@@ -12,6 +12,7 @@ public class PercentilesAggregator {
 
         private long cumulativeCount = 0;
 
+        private long minValue = Long.MAX_VALUE;
         private long maxValue = 0;
 
         private final Percentiles percentiles = new Percentiles(POINTS);
@@ -26,11 +27,12 @@ public class PercentilesAggregator {
         public ToPercentiles(final long totalValues) {
             this.totalValues = totalValues;
             stepSize = 100.0 / POINTS;
-            nextPercentile = stepSize;
+            nextPercentile = 0;
         }
 
         @Override
         public void accept(final long duration, final long count) {
+            minValue = Math.min(minValue, duration);
             maxValue = duration;
 
             cumulativeCount += count;
@@ -54,6 +56,7 @@ public class PercentilesAggregator {
 
         public void collect(final LongLongHashMap map) {
             map.forEachOrdered(this);
+            percentiles.put("0.000", minValue);
             percentiles.put("100.000", maxValue);
         }
 

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -66,6 +66,7 @@ public class GnuplotFileGenerator implements Appender {
 
         // appendfln(result, "set xrange [-1:1]");
         appendfln(result, "set boxwidth 0.5");
+        // appendfln(result, "set boxwidth 3600");
 
         appendfln(result, "set style fill transparent solid 0.5");
 
@@ -75,7 +76,7 @@ public class GnuplotFileGenerator implements Appender {
         // render images when there are not data points on it.
         appendf(result, "-1 with lines notitle");
 
-        LOGGER.debug("{}", result);
+        LOGGER.info("{}", result);
 
         return result.toString();
     }

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotLineType.java

@@ -5,6 +5,8 @@ public enum GnuplotLineType {
 
     Bar("boxes"),
 
+    BOX("candlesticks"),
+
     Points("points");
 
     private String gnuplotLineType;

pdb-plotting/src/main/java/org/lucares/recommind/logs/LineStyle.java

@@ -13,13 +13,7 @@ public class LineStyle {
     }
 
     private String asGnuplotLineStyle(final String colorHex) {
-        // TODO revert
+        return String.format("linetype rgb \"#%s\" ", //
-//        return String.format("lt rgb \"#%s\" dt %s ", //
-//                colorHex, //
-//                dashType.toGnuplotDashType()//
-//        );
-
-        return String.format("lt rgb \"#%s\" ", //
                 colorHex//
         );
     }

pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java

@@ -5,6 +5,7 @@ import java.util.List;
 import org.lucares.pdb.plot.api.Aggregate;
 import org.lucares.pdb.plot.api.AggregateHandlerCollection;
 import org.lucares.pdb.plot.api.BarChartHandler;
+import org.lucares.pdb.plot.api.BoxChartHandler;
 import org.lucares.pdb.plot.api.CumulativeDistributionHandler;
 import org.lucares.pdb.plot.api.HistogramHandler;
 import org.lucares.pdb.plot.api.Interval;
@@ -62,6 +63,9 @@ class PlotSettingsTransformer {
             case BAR:
                 aggregateHandlerCollection.addAggregateHandler(new BarChartHandler());
                 break;
+            case BOX:
+                aggregateHandlerCollection.addAggregateHandler(new BoxChartHandler());
+                break;
             default:
                 throw new IllegalStateException("unhandled enum: " + aggregate);
             }