add histogram plots

build.gradle

@@ -22,7 +22,7 @@ ext {
 
     javaVersion=12
     
-    version_log4j2= '2.13.0'
+    version_log4j2= '2.12.1'
     version_spring = '2.2.2.RELEASE'
     
     lib_antlr = "org.antlr:antlr4:4.7.2"

pdb-js/src/app/plot.service.ts

@@ -17,7 +17,7 @@ export class PlotService {
     this.plotTypes.push(new PlotType("HEATMAP", "Heatmap", "heatmap", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("CONTOUR", "Contour", "contour-chart", false, DataType.Time, DataType.Duration));
     this.plotTypes.push(new PlotType("CUM_DISTRIBUTION", "Cumulative Distribution", "cumulative-distribution-chart", true, DataType.Percent, DataType.Duration));
-    this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", false, DataType.Group, DataType.Duration));
+    this.plotTypes.push(new PlotType("HISTOGRAM", "Histogram", "histogram", true, DataType.HistogramBin, DataType.HistogramCount));
     this.plotTypes.push(new PlotType("RIDGELINES", "Ridgelines", "ridgelines", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("QQ", "Quantile-Quantile", "quantile-quantile", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("PARALLEL", "Parallel Requests", "parallel-requests-chart", true, DataType.Time, DataType.Count));
@@ -26,6 +26,8 @@ export class PlotService {
     this.plotTypes.push(new PlotType("PIE", "Pie", "pie-chart", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("BAR", "Bar", "bar-chart", false, DataType.Other, DataType.Other));
     this.plotTypes.push(new PlotType("STEP_FIT", "Step Fit", "step-fit", false, DataType.Other, DataType.Other));
+    this.plotTypes.push(new PlotType("LAG", "Lag", "lag-plot", false, DataType.Other, DataType.Other));
+    this.plotTypes.push(new PlotType("ACF", "ACF", "acf-plot", false, DataType.Other, DataType.Other));
   }
   
   ngOnInit() {
@@ -125,6 +127,8 @@ export enum DataType {
     Count,
     Group,
     Metric,
+    HistogramBin,
+    HistogramCount,
     Other
 }
 

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Aggregate.java

@@ -15,4 +15,6 @@ public enum Aggregate {
      * @see https://serialmentor.com/dataviz/ecdf-qq.html
      */
     CUM_DISTRIBUTION,
+
+    HISTOGRAM
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramAggregator.java

@@ -0,0 +1,101 @@
+package org.lucares.pdb.plot.api;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+
+import org.lucares.collections.LongLongConsumer;
+import org.lucares.collections.LongLongHashMap;
+
+public class HistogramAggregator implements CustomAggregator {
+
+    private final static class ToBins implements LongLongConsumer {
+
+        final LongLongHashMap bins;
+
+        private final int binWidth;
+
+        public ToBins(final int numBins, final int binWidth) {
+            this.binWidth = binWidth;
+            // use large initial capacity to reduce likelihood of collisions in hashmap
+            bins = new LongLongHashMap(numBins * 5, 0.75);
+        }
+
+        @Override
+        public void accept(final long key, final long value) {
+            final long bin = binWidth * (key / binWidth);
+            bins.compute(bin, 0, oldValue -> oldValue + value);
+        }
+
+        public LongLongHashMap getBins() {
+            return bins;
+        }
+    }
+
+    // the rather large initial capacity should prevent too many grow&re-hash phases
+    private final LongLongHashMap map = new LongLongHashMap(5_000, 0.75);
+    private long min;
+    private long max;
+    private final Path tmpDir;
+    private final PlotSettings plotSettings;
+
+    public HistogramAggregator(final Path tmpDir, final PlotSettings plotSettings) {
+        this.tmpDir = tmpDir;
+        this.plotSettings = plotSettings;
+    }
+
+    @Override
+    public void addValue(final boolean valueIsInYRange, final long epochMilli, final long value) {
+        map.compute(value, 0, l -> l + 1);
+        min = min < value ? min : value;
+        max = max > value ? max : value;
+    }
+
+    @Override
+    public AggregatedData getAggregatedData() throws IOException {
+        final char separator = ',';
+        final char newline = '\n';
+
+        final int numBins = plotSettings.getWidth() / 3;
+        final int binWidth = Math.max((int) (max) / numBins, 1);
+
+        final ToBins toBins = new ToBins(numBins, binWidth);
+        map.forEachOrdered(toBins);
+
+        final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
+        try (final Writer output = new BufferedWriter(
+                new OutputStreamWriter(new FileOutputStream(dataFile), StandardCharsets.US_ASCII));) {
+            final StringBuilder data = new StringBuilder();
+            if (map.size() > 0) {
+                // compute the percentiles
+                final LongLongHashMap bins = toBins.getBins();
+                for (int i = 0; i < numBins; i++) {
+                    final int bin = i * binWidth;
+                    final long value = bins.get(bin, 0);
+                    data.append(bin);
+                    data.append(separator);
+                    data.append(value);
+                    data.append(newline);
+
+                }
+
+            }
+            output.write(data.toString());
+            System.out.println(data.toString());
+
+        }
+        final AggregatedData result = new AggregatedData("histogram", dataFile);
+        return result;
+    }
+
+    @Override
+    public Aggregate getType() {
+        return Aggregate.HISTOGRAM;
+    }
+
+}

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/HistogramHandler.java

@@ -0,0 +1,73 @@
+package org.lucares.pdb.plot.api;
+
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Optional;
+
+import org.lucares.recommind.logs.AxisSettings;
+import org.lucares.recommind.logs.AxisSettings.Type;
+import org.lucares.recommind.logs.DataSeries;
+import org.lucares.recommind.logs.GnuplotAxis;
+import org.lucares.recommind.logs.GnuplotSettings;
+import org.lucares.recommind.logs.LineStyle;
+
+public class HistogramHandler extends AggregateHandler {
+
+    @Override
+    Type getAxisType(final GnuplotAxis axis) {
+        switch (axis) {
+        case X1:
+        case X2:
+            return Type.HistogramBin;
+        case Y1:
+        case Y2:
+            return Type.HistogramCount;
+        default:
+            throw new IllegalArgumentException("Unexpected value: " + axis);
+        }
+    }
+
+    @Override
+    Aggregate getAggregateType() {
+        return Aggregate.HISTOGRAM;
+    }
+
+    @Override
+    AxisSettings createXAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
+        final AxisSettings result = new AxisSettings();
+        result.setLabel("Histogram - Duration in ms");
+        result.setType(Type.HistogramBin);
+        result.setAxis(getxAxis());
+        result.setTicsEnabled(true);
+        return result;
+    }
+
+    @Override
+    AxisSettings createYAxisSettings(final GnuplotSettings settings, final Collection<DataSeries> dataSeries) {
+        final AxisSettings result = new AxisSettings();
+        result.setLabel("Histogram - Count");
+        result.setType(Type.HistogramCount);
+        result.setAxis(getyAxis());
+        result.setTicsEnabled(true);
+        result.setFrom("0");
+        return result;
+    }
+
+    @Override
+    void addPlot(final StringBuilder result, final AggregatedData aggregatedData, final LineStyle lineStyle,
+            final Optional<String> title) {
+        appendfln(result, "'%s' using 1:2 %s with lines axes %s lw 2 %s, \\", //
+                aggregatedData.getDataFile().getAbsolutePath(), //
+                gnuplotTitle(title), //
+                gnuplotXYAxis(), //
+                lineStyle.darker()//
+        );
+    }
+
+    @Override
+    CustomAggregator createCustomAggregator(final Path tmpDir, final PlotSettings plotSettings,
+            final long fromEpochMilli, final long toEpochMilli) {
+        return new HistogramAggregator(tmpDir, plotSettings);
+    }
+
+}

pdb-plotting/src/main/java/org/lucares/recommind/logs/AxisSettings.java

@@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 public class AxisSettings {
 
     public enum Type {
-        Number, Time, Duration, Percent
+        Number, Time, Duration, Percent, HistogramBin, HistogramCount
     }
 
     private String format = "";
@@ -38,7 +38,7 @@ public class AxisSettings {
         return format;
     }
 
-    public void setFormat(String format) {
+    public void setFormat(final String format) {
         this.format = format;
     }
 
@@ -46,7 +46,7 @@ public class AxisSettings {
         return label;
     }
 
-    public void setLabel(String label) {
+    public void setLabel(final String label) {
         this.label = label;
     }
 
@@ -54,7 +54,7 @@ public class AxisSettings {
         return rotateLabel;
     }
 
-    public void setRotateLabel(int rotateLabel) {
+    public void setRotateLabel(final int rotateLabel) {
         this.rotateLabel = rotateLabel;
     }
 
@@ -62,7 +62,7 @@ public class AxisSettings {
         return from;
     }
 
-    public void setFrom(String from) {
+    public void setFrom(final String from) {
         this.from = from;
     }
 
@@ -70,7 +70,7 @@ public class AxisSettings {
         return to;
     }
 
-    public void setTo(String to) {
+    public void setTo(final String to) {
         this.to = to;
     }
 
@@ -78,7 +78,7 @@ public class AxisSettings {
         return type;
     }
 
-    public void setType(Type type) {
+    public void setType(final Type type) {
         this.type = type;
     }
 
@@ -86,11 +86,11 @@ public class AxisSettings {
         return axis;
     }
 
-    public void setAxis(GnuplotAxis axis) {
+    public void setAxis(final GnuplotAxis axis) {
         this.axis = axis;
     }
 
-    public void setTicIncrement(double ticIncrement) {
+    public void setTicIncrement(final double ticIncrement) {
         this.ticIncrement = ticIncrement;
     }
 
@@ -98,7 +98,7 @@ public class AxisSettings {
         return ticIncrement;
     }
 
-    public void setTicsEnabled(boolean ticsEnabled) {
+    public void setTicsEnabled(final boolean ticsEnabled) {
         this.ticsEnabled = ticsEnabled;
     }
 
@@ -106,7 +106,7 @@ public class AxisSettings {
         return ticsEnabled;
     }
 
-    public void setLogscale(boolean logscale) {
+    public void setLogscale(final boolean logscale) {
         this.logscale = logscale;
     }
 
@@ -114,7 +114,7 @@ public class AxisSettings {
         return logscale;
     }
 
-    public void setTics(List<String> ticsLabels) {
+    public void setTics(final List<String> ticsLabels) {
         this.ticsLabels = ticsLabels;
     }
 
@@ -122,8 +122,8 @@ public class AxisSettings {
         return ticsLabels;
     }
 
-    public String toGnuplotDefinition(boolean renderLabels) {
+    public String toGnuplotDefinition(final boolean renderLabels) {
-        StringBuilder result = new StringBuilder();
+        final StringBuilder result = new StringBuilder();
         if (type == Type.Time) {
             appendfln(result, "set %sdata time", axis);
         }
@@ -171,10 +171,10 @@ public class AxisSettings {
 
     @Override
     public String toString() {
-        ObjectMapper mapper = new ObjectMapper();
+        final ObjectMapper mapper = new ObjectMapper();
         try {
             return mapper.writeValueAsString(this);
-        } catch (JsonProcessingException e) {
+        } catch (final JsonProcessingException e) {
             return e.getMessage();
         }
     }

pdb-plotting/src/main/java/org/lucares/recommind/logs/GnuplotFileGenerator.java

@@ -24,7 +24,7 @@ public class GnuplotFileGenerator implements Appender {
         appendfln(result, "set timefmt '%s'", settings.getTimefmt());
 
         final List<AxisSettings> xAxisDefinitions = settings.getAggregates().getXAxisDefinitions(settings, dataSeries);
-        for (AxisSettings axisSettings : xAxisDefinitions) {
+        for (final AxisSettings axisSettings : xAxisDefinitions) {
             appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels()));
         }
 
@@ -34,10 +34,9 @@ public class GnuplotFileGenerator implements Appender {
             // Workaround is to explicitly specify the y-axis range.
             // We choose a range for which no ticks are defined. This creates an empty
             // y-axis.
-            yAxisDefinitions.forEach(s -> s.setFrom("0"));
             yAxisDefinitions.forEach(s -> s.setFrom("-1"));
         }
-        for (AxisSettings axisSettings : yAxisDefinitions) {
+        for (final AxisSettings axisSettings : yAxisDefinitions) {
             appendln(result, axisSettings.toGnuplotDefinition(settings.isRenderLabels()));
         }
 

pdb-ui/src/main/java/org/lucares/pdbui/PlotSettingsTransformer.java

@@ -4,6 +4,7 @@ import org.lucares.pdb.plot.api.Aggregate;
 import org.lucares.pdb.plot.api.AggregateHandlerCollection;
 import org.lucares.pdb.plot.api.AxisScale;
 import org.lucares.pdb.plot.api.CumulativeDistributionHandler;
+import org.lucares.pdb.plot.api.HistogramHandler;
 import org.lucares.pdb.plot.api.ParallelRequestsAggregate;
 import org.lucares.pdb.plot.api.PlotSettings;
 import org.lucares.pdb.plot.api.ScatterAggregateHandler;
@@ -55,11 +56,11 @@ class PlotSettingsTransformer {
         throw new IllegalStateException("unhandled enum value: " + yRangeUnit);
     }
 
-    static AggregateHandlerCollection toAggregateInternal(TimeRangeUnitInternal yRangeUnit, AxisScale yAxisScale,
+    static AggregateHandlerCollection toAggregateInternal(final TimeRangeUnitInternal yRangeUnit,
-            final Iterable<Aggregate> aggregates) {
+            final AxisScale yAxisScale, final Iterable<Aggregate> aggregates) {
         final AggregateHandlerCollection aggregateHandlerCollection = new AggregateHandlerCollection();
 
-        for (Aggregate aggregate : aggregates) {
+        for (final Aggregate aggregate : aggregates) {
 
             switch (aggregate) {
             case CUM_DISTRIBUTION:
@@ -78,6 +79,9 @@ class PlotSettingsTransformer {
                     aggregateHandlerCollection.add(new ScatterAggregateHandler());
                 }
                 break;
+            case HISTOGRAM:
+                aggregateHandlerCollection.add(new HistogramHandler());
+                break;
             default:
                 throw new IllegalStateException("unhandled enum: " + aggregate);
             }