add percentile information if available

pdb-js/src/app/app.module.ts

@@ -19,6 +19,7 @@ import { MatFormFieldModule } from '@angular/material/form-field';
 import { MatInputModule } from '@angular/material/input';
 import {MatProgressBarModule} from '@angular/material/progress-bar';
 import {MatProgressSpinnerModule} from '@angular/material/progress-spinner';
+import {MatRadioModule} from '@angular/material/radio';
 import {MatSnackBarModule} from '@angular/material/snack-bar';
 import {MatTooltipModule} from '@angular/material/tooltip';
 import { YAxisDefinitionComponent } from './y-axis-definition/y-axis-definition.component';
@@ -56,6 +57,7 @@ import { ImageToggleComponent } from './image-toggle/image-toggle.component';
     MatCheckboxModule,
     MatFormFieldModule,
     MatInputModule,
+    MatRadioModule,
     MatProgressBarModule,
     MatProgressSpinnerModule,
     MatSelectModule,

pdb-js/src/app/plot-details/plot-details.component.html

@@ -1,28 +1,59 @@
+<mat-radio-group [(ngModel)]="valueFormat" aria-label="Value format">
+  <mat-radio-button value="time">Time</mat-radio-button>
+  <mat-radio-button value="numbers">Numbers</mat-radio-button>
+</mat-radio-group>
+
 <table  class="gallery-item-details">
   <tr>
     <th>Name</th>
     <th>Type</th>
     <th>Values</th>
     <th>Avg</th>
+    <td *ngFor="let label of percentilesToPlot.keys()">{{label}}</td>
+    <td>Max</td>
   </tr>
   <tr *ngFor="let stat of stats.dataSeriesStats">
     <td>{{ stat.name }}</td>
     <td><div class="{{ pointTypeClass(stat.dashTypeAndColor) }}" title="{{ stat.name }}"></div></td>
     <td>{{ stat.values }}</td>
-    <td>{{ utils.formatMs(stat.average) }}</td>
+    <td>{{ utils.format(stat.average, valueFormat) }}</td>
-  </tr>
+    <td *ngFor="let key of percentilesToPlot.keys()">{{utils.format(stat.percentiles[percentilesToPlot.get(key)], valueFormat)}}</td>
-</table>
+    <td>{{ utils.format(stat.maxValue, valueFormat)}}</td>
-<table  class="gallery-item-details-matrix">
-  <tr>
-    <th></th>
-    <th *ngFor="let statsCol of stats.dataSeriesStats">
-      <div class="{{ pointTypeClass(statsCol.dashTypeAndColor) }}" title="{{ statsCol.name }}"></div>
-    </th>
-  </tr>
-  <tr *ngFor="let statsRow of stats.dataSeriesStats">
-    <td><div class="{{ pointTypeClass(statsRow.dashTypeAndColor) }}" title="{{ statsRow.name }}"></div></td>
-    <td *ngFor="let statsCol of stats.dataSeriesStats">
-      {{ utils.toPercent(statsRow.average / statsCol.average)  }}
-    </td>
   </tr>
 </table>
+<div *ngIf="stats.dataSeriesStats.length > 1">
+  <h2>Compare Averages</h2>
+  <table  class="gallery-item-details-matrix">
+    <tr>
+      <th></th>
+      <th *ngFor="let statsCol of stats.dataSeriesStats">
+        <div class="{{ pointTypeClass(statsCol.dashTypeAndColor) }}" title="{{ statsCol.name }}"></div>
+      </th>
+    </tr>
+    <tr *ngFor="let statsRow of stats.dataSeriesStats">
+      <td><div class="{{ pointTypeClass(statsRow.dashTypeAndColor) }}" title="{{ statsRow.name }}"></div></td>
+      <td *ngFor="let statsCol of stats.dataSeriesStats">
+        {{ utils.toPercent(statsRow.average / statsCol.average)  }}
+      </td>
+    </tr>
+  </table>
+
+  <h2>Compare Percentiles</h2>
+  <div *ngFor="let p of percentilesToPlot.keys()">
+    <h3>{{p}} percentile</h3>
+    <table  class="gallery-item-details-matrix">
+      <tr>
+        <th></th>
+        <th *ngFor="let statsCol of stats.dataSeriesStats">
+          <div class="{{ pointTypeClass(statsCol.dashTypeAndColor) }}" title="{{ statsCol.name }}"></div>
+        </th>
+      </tr>
+      <tr *ngFor="let statsRow of stats.dataSeriesStats">
+        <td><div class="{{ pointTypeClass(statsRow.dashTypeAndColor) }}" title="{{ statsRow.name }}"></div></td>
+        <td *ngFor="let statsCol of stats.dataSeriesStats">
+          {{ utils.toPercent(statsRow.percentiles[percentilesToPlot.get(p)] / statsCol.percentiles[percentilesToPlot.get(p)])  }}
+        </td>
+      </tr>
+    </table>
+  </div>
+</div>

pdb-js/src/app/plot-details/plot-details.component.scss

@@ -28,3 +28,8 @@
 .plot-details-plotType_e51e10 {background-position-y: -40px;}
 .plot-details-plotType_57a1c2 {background-position-y: -48px;}
 .plot-details-plotType_bd36c2 {background-position-y: -56px;}
+
+
+.gallery-item-details td {
+  white-space: pre;
+}

pdb-js/src/app/plot-details/plot-details.component.ts

@@ -12,10 +12,38 @@ export class PlotDetailsComponent {
   @Input()
   stats: PlotResponseStats;
 
+  hasPercentiles = false;
+  
+  valueFormat = "time";
+
+  percentilesToPlot : Map<string,string> = new Map();
+
   constructor(public utils: UtilService){
 
   }
 
+  ngOnInit() {
+    this.hasPercentiles = false;
+    this.percentilesToPlot.clear();
+    for (let i = 0; i < this.stats.dataSeriesStats.length; i++)
+    {
+      const stat = this.stats.dataSeriesStats[i];
+      if (stat.percentiles.hasOwnProperty("50.000"))
+      {
+        this.hasPercentiles = true;
+        this.percentilesToPlot.set('median','50.000');
+        this.percentilesToPlot.set('75th','75.000');
+        this.percentilesToPlot.set('95th','95.000');
+        this.percentilesToPlot.set('99th','99.000');
+        break;
+      }
+    }
+  }
+
+  percentile(value: number): string {
+    return this.utils.format(value, this.valueFormat);
+  }
+
   pointTypeClass(typeAndColor: DashTypeAndColor): string {
     return "plot-details-plotType"
       +" plot-details-plotType_"+typeAndColor.pointType

pdb-js/src/app/plot-view/plot-view.component.scss

@@ -19,4 +19,5 @@ img {
   bottom: 5px;
   background-color: white;
   box-shadow: 5px 5px 10px 0px #e0e0e0;
+  overflow: auto;
 }

pdb-js/src/app/plot.service.ts

@@ -249,6 +249,7 @@ export class DataSeriesStats {
   average : number;
   plottedValues : number;
   dashTypeAndColor: DashTypeAndColor;
+  percentiles: Map<string, number>
 }
 
 export class DashTypeAndColor {

pdb-js/src/app/utils.service.ts

@@ -9,6 +9,14 @@ export class UtilService {
     constructor() {
     }
 
+    format(value: number, type: string) {
+        if (type == "time"){
+            return this.formatMs(value);
+        } else {
+            return ""+value;
+        }
+    }
+
     formatMs(valueInMs):string {
         const ms = Math.floor(valueInMs % 1000);
         const s = Math.floor((valueInMs / 1000) % 60);

pdb-js/src/app/visualization-page/visualization-page.component.scss

@@ -48,6 +48,7 @@
 
 #filters {
   grid-area: filters;
+  overflow: auto;
 }
 #filterpanel {
   background-color: #f8f8f8;/*#fafafa;*/

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/AggregatorCollection.java

@@ -3,6 +3,7 @@ package org.lucares.pdb.plot.api;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 
 import org.lucares.pdb.api.Tags;
@@ -27,4 +28,15 @@ public class AggregatorCollection {
         return Optional.ofNullable(aggregators.get(type));
     }
 
+    @SuppressWarnings("unchecked")
+    public <T extends CustomAggregator> Optional<T> getAggregator(final Class<T> aggregatorType) {
+
+        for (final CustomAggregator aggregator : aggregators.values()) {
+            if (Objects.equals(aggregator.getClass(), aggregatorType)) {
+                return Optional.of((T) aggregator);
+            }
+        }
+
+        return Optional.empty();
+    }
 }

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/CumulativeDistributionCustomAggregator.java

@@ -8,7 +8,7 @@ import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.util.LinkedHashMap;
+import java.util.Locale;
 
 import org.lucares.collections.LongLongConsumer;
 import org.lucares.collections.LongLongHashMap;
@@ -24,7 +24,7 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
 
         private long maxValue = 0;
 
-        private final LinkedHashMap<Double, Long> percentiles = new LinkedHashMap<>(POINTS);
+        private final Percentiles percentiles = new Percentiles(POINTS);
 
         private final double stepSize;
 
@@ -49,7 +49,8 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
             if (newPercentile >= nextPercentile) {
                 double currentPercentile = lastPercentile + stepSize;
                 while (currentPercentile <= newPercentile) {
-                    percentiles.put(currentPercentile, duration);
+                    final String percentile = String.format(Locale.US, "%.3f", currentPercentile);
+                    percentiles.put(percentile, duration);
                     currentPercentile += stepSize;
                 }
                 nextPercentile = currentPercentile;
@@ -61,10 +62,15 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
             return maxValue;
         }
 
-        public LinkedHashMap<Double, Long> getPercentiles() {
+        public Percentiles getPercentiles() {
             return percentiles;
         }
 
+        public void collect(final LongLongHashMap map) {
+            map.forEachOrdered(this);
+            percentiles.put("100.000", maxValue);
+        }
+
     }
 
     // the rather large initial capacity should prevent too many grow&re-hash phases
@@ -84,14 +90,27 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
         totalValues++;
     }
 
+    public Percentiles getPercentiles() {
+        final long start = System.nanoTime();
+
+        final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
+        toPercentiles.collect(map);
+
+        final Percentiles result = toPercentiles.getPercentiles();
+        System.out.println("getPercentiles took: " + (System.nanoTime() - start) / 1_000_000.0 + " ms");
+        return result;
+    }
+
     @Override
     public AggregatedData getAggregatedData() {
         try {
             final char separator = ',';
             final char newline = '\n';
 
+            final long start = System.nanoTime();
             final ToPercentiles toPercentiles = new ToPercentiles(totalValues);
-            map.forEachOrdered(toPercentiles);
+            toPercentiles.collect(map);
+            System.out.println("getAggregated took: " + (System.nanoTime() - start) / 1_000_000.0 + " ms");
 
             final File dataFile = File.createTempFile("data", ".dat", tmpDir.toFile());
             try (final Writer output = new BufferedWriter(
@@ -107,12 +126,6 @@ public class CumulativeDistributionCustomAggregator implements CustomAggregator
                         data.append(value);
                         data.append(newline);
                     });
-
-                    final long maxValue = toPercentiles.getMaxValue();
-                    data.append(100);
-                    data.append(separator);
-                    data.append(maxValue);
-                    data.append(newline);
                 }
                 output.write(data.toString());
 

pdb-plotting/src/main/java/org/lucares/pdb/plot/api/Percentiles.java

@@ -0,0 +1,32 @@
+package org.lucares.pdb.plot.api;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Maps percentiles to their value. E.g.
+ * 
+ * <pre>
+ * {"50.00": 123, "75.00": 567}
+ * </pre>
+ * 
+ * This class uses Strings for the precentiles instead of doubles, because
+ * doubles are bad keys for maps.
+ */
+public class Percentiles extends LinkedHashMap<String, Long> {
+
+    private static final long serialVersionUID = 4957667781086113971L;
+
+    public Percentiles() {
+        super(0);
+    }
+
+    public Percentiles(final int initialSize) {
+        super(initialSize);
+    }
+
+    public Percentiles(final Map<String, Long> percentiles) {
+        super(percentiles.size());
+        putAll(percentiles);
+    }
+}

pdb-plotting/src/main/java/org/lucares/recommind/logs/DataSeries.java

@@ -8,6 +8,7 @@ import java.util.Map;
 
 import org.lucares.pdb.plot.api.AggregatorCollection;
 import org.lucares.pdb.plot.api.Limit;
+import org.lucares.pdb.plot.api.Percentiles;
 
 public interface DataSeries {
     public static final Comparator<? super DataSeries> BY_NUMBER_OF_VALUES = (a, b) -> {
@@ -35,6 +36,8 @@ public interface DataSeries {
 
     public double getAverage();
 
+    public Percentiles getPercentiles();
+
     public void setStyle(LineStyle style);
 
     public LineStyle getStyle();
@@ -114,5 +117,4 @@ public interface DataSeries {
 
         return result;
     }
-
 }

pdb-plotting/src/main/java/org/lucares/recommind/logs/FileBackedDataSeries.java

@@ -1,6 +1,8 @@
 package org.lucares.recommind.logs;
 
 import org.lucares.pdb.plot.api.AggregatorCollection;
+import org.lucares.pdb.plot.api.CumulativeDistributionCustomAggregator;
+import org.lucares.pdb.plot.api.Percentiles;
 
 public class FileBackedDataSeries implements DataSeries {
 
@@ -55,11 +57,19 @@ public class FileBackedDataSeries implements DataSeries {
 
     @Override
     public double getAverage() {
-        return csvSummary.getStatsAverage();
+        return Math.round(csvSummary.getStatsAverage() * 10.0) / 10.0;
     }
 
     @Override
     public AggregatorCollection getAggregators() {
         return csvSummary.getAggregators();
     }
+
+    @Override
+    public Percentiles getPercentiles() {
+        return csvSummary.getAggregators()//
+                .getAggregator(CumulativeDistributionCustomAggregator.class)//
+                .map(CumulativeDistributionCustomAggregator::getPercentiles)//
+                .orElse(new Percentiles());
+    }
 }

pdb-ui/src/main/java/org/lucares/pdbui/domain/DataSeriesStats.java

@@ -2,74 +2,82 @@ package org.lucares.pdbui.domain;
 
 import java.util.Collection;
 
+import org.lucares.pdb.plot.api.Percentiles;
+
 public class DataSeriesStats {
-    private final int values;
+  private final int values;
-    private final long maxValue;
+  private final long maxValue;
-    private final double average;
+  private final double average;
-    private final String name;
+  private final String name;
-    private final StyleAndColor dashTypeAndColor;
+  private final StyleAndColor dashTypeAndColor;
+  private Percentiles percentiles;
 
-    public DataSeriesStats(final int values, final long maxValue, final double average) {
+  public DataSeriesStats(final int values, final long maxValue, final double average) {
-        this.name = "<noName>";
+    this.name = "<noName>";
-        this.dashTypeAndColor = new StyleAndColor("000", 0);
+    this.dashTypeAndColor = new StyleAndColor("000", 0);
-        this.values = values;
+    this.values = values;
-        this.maxValue = maxValue;
+    this.maxValue = maxValue;
-        this.average = average;
+    this.average = average;
+  }
+
+  public DataSeriesStats(final String name, final StyleAndColor dashTypeAndColor, final int values, final long maxValue,
+      final double average, final Percentiles percentiles) {
+    this.name = name;
+    this.dashTypeAndColor = dashTypeAndColor;
+    this.values = values;
+    this.maxValue = maxValue;
+    this.average = average;
+    this.percentiles = percentiles;
+  }
+
+  /**
+   * The number of values in the date range, without applying the y-range.
+   *
+   * @return total number of values
+   */
+  public int getValues() {
+    return values;
+  }
+
+  public StyleAndColor getDashTypeAndColor() {
+    return dashTypeAndColor;
+  }
+
+  public long getMaxValue() {
+    return maxValue;
+  }
+
+  public double getAverage() {
+    return average;
+  }
+
+  public Percentiles getPercentiles() {
+    return percentiles;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public String toString() {
+    return "[name=" + name + ", dashTypeAndColor=" + dashTypeAndColor + ", values=" + values + ", maxValue=" + maxValue
+        + ", average=" + average + "]";
+  }
+
+  public static double average(final Collection<DataSeriesStats> stats) {
+    long n = 0;
+    double average = 0;
+
+    for (final DataSeriesStats stat : stats) {
+      final int newValues = stat.getValues();
+      final double newAverage = stat.getAverage();
+      if (newValues > 0) {
+        average = (average * n + newAverage * newValues) / (n + newValues);
+        n += newValues;
+      }
     }
 
-    public DataSeriesStats(final String name, final StyleAndColor dashTypeAndColor, final int values,
+    return average;
-            final long maxValue, final double average) {
+  }
-        this.name = name;
-        this.dashTypeAndColor = dashTypeAndColor;
-        this.values = values;
-        this.maxValue = maxValue;
-        this.average = average;
-    }
-
-    /**
-     * The number of values in the date range, without applying the y-range.
-     *
-     * @return total number of values
-     */
-    public int getValues() {
-        return values;
-    }
-
-    public StyleAndColor getDashTypeAndColor() {
-        return dashTypeAndColor;
-    }
-
-    public long getMaxValue() {
-        return maxValue;
-    }
-
-    public double getAverage() {
-        return average;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    @Override
-    public String toString() {
-        return "[name=" + name + ", dashTypeAndColor=" + dashTypeAndColor + ", values=" + values + ", maxValue="
-                + maxValue + ", average=" + average + "]";
-    }
-
-    public static double average(final Collection<DataSeriesStats> stats) {
-        long n = 0;
-        double average = 0;
-
-        for (final DataSeriesStats stat : stats) {
-            final int newValues = stat.getValues();
-            final double newAverage = stat.getAverage();
-            if (newValues > 0) {
-                average = (average * n + newAverage * newValues) / (n + newValues);
-                n += newValues;
-            }
-        }
-
-        return average;
-    }
 }

pdb-ui/src/main/java/org/lucares/pdbui/domain/PlotResponseStats.java

@@ -6,84 +6,84 @@ import java.util.List;
 import org.lucares.recommind.logs.DataSeries;
 
 public class PlotResponseStats {
-    private long maxValue;
+  private long maxValue;
 
-    private int values;
+  private int values;
 
-    private double average;
+  private double average;
 
-    private List<DataSeriesStats> dataSeriesStats;
+  private List<DataSeriesStats> dataSeriesStats;
 
-    public PlotResponseStats() {
+  public PlotResponseStats() {
-        super();
+    super();
+  }
+
+  public PlotResponseStats(final long maxValue, final int values, final double average,
+      final List<DataSeriesStats> dataSeriesStats) {
+
+    this.maxValue = maxValue;
+    this.values = values;
+    this.average = average;
+    this.dataSeriesStats = dataSeriesStats;
+  }
+
+  public long getMaxValue() {
+    return maxValue;
+  }
+
+  public void setMaxValue(final long maxValue) {
+    this.maxValue = maxValue;
+  }
+
+  public int getValues() {
+    return values;
+  }
+
+  public void setValues(final int values) {
+    this.values = values;
+  }
+
+  public double getAverage() {
+    return average;
+  }
+
+  public void setAverage(final double average) {
+    this.average = average;
+  }
+
+  public List<DataSeriesStats> getDataSeriesStats() {
+    return dataSeriesStats;
+  }
+
+  public void setDataSeriesStats(final List<DataSeriesStats> dataSeriesStats) {
+    this.dataSeriesStats = dataSeriesStats;
+  }
+
+  @Override
+  public String toString() {
+    return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + ", average=" + average
+        + ", dataSeriesStats=" + dataSeriesStats + "]";
+  }
+
+  public static PlotResponseStats fromDataSeries(final List<DataSeries> dataSeries) {
+
+    int values = 0;
+    long maxValue = 0;
+    final List<DataSeriesStats> dataSeriesStats = new ArrayList<>();
+
+    for (final DataSeries dataSerie : dataSeries) {
+      values += dataSerie.getValues();
+      maxValue = Math.max(maxValue, dataSerie.getMaxValue());
+
+      final StyleAndColor lineStyle = new StyleAndColor(dataSerie.getStyle().getColor().getColor(),
+          dataSerie.getStyle().getPointType());
+
+      dataSeriesStats.add(new DataSeriesStats(dataSerie.getTitle(), lineStyle, dataSerie.getValues(),
+          dataSerie.getMaxValue(), dataSerie.getAverage(), dataSerie.getPercentiles()));
     }
 
-    public PlotResponseStats(final long maxValue, final int values, final double average,
+    final double average = Math.round(DataSeriesStats.average(dataSeriesStats));
-            final List<DataSeriesStats> dataSeriesStats) {
 
-        this.maxValue = maxValue;
+    return new PlotResponseStats(maxValue, values, average, dataSeriesStats);
-        this.values = values;
+  }
-        this.average = average;
-        this.dataSeriesStats = dataSeriesStats;
-    }
-
-    public long getMaxValue() {
-        return maxValue;
-    }
-
-    public void setMaxValue(final long maxValue) {
-        this.maxValue = maxValue;
-    }
-
-    public int getValues() {
-        return values;
-    }
-
-    public void setValues(final int values) {
-        this.values = values;
-    }
-
-    public double getAverage() {
-        return average;
-    }
-
-    public void setAverage(final double average) {
-        this.average = average;
-    }
-
-    public List<DataSeriesStats> getDataSeriesStats() {
-        return dataSeriesStats;
-    }
-
-    public void setDataSeriesStats(final List<DataSeriesStats> dataSeriesStats) {
-        this.dataSeriesStats = dataSeriesStats;
-    }
-
-    @Override
-    public String toString() {
-        return "PlotResponseStats [maxValue=" + maxValue + ", values=" + values + ", average=" + average
-                + ", dataSeriesStats=" + dataSeriesStats + "]";
-    }
-
-    public static PlotResponseStats fromDataSeries(final List<DataSeries> dataSeries) {
-
-        int values = 0;
-        long maxValue = 0;
-        final List<DataSeriesStats> dataSeriesStats = new ArrayList<>();
-
-        for (final DataSeries dataSerie : dataSeries) {
-            values += dataSerie.getValues();
-            maxValue = Math.max(maxValue, dataSerie.getMaxValue());
-
-            final StyleAndColor lineStyle = new StyleAndColor(dataSerie.getStyle().getColor().getColor(),
-                    dataSerie.getStyle().getPointType());
-
-            dataSeriesStats.add(new DataSeriesStats(dataSerie.getTitle(), lineStyle, dataSerie.getValues(),
-                    dataSerie.getMaxValue(), dataSerie.getAverage()));
-        }
-
-        final double average = Math.round(DataSeriesStats.average(dataSeriesStats));
-
-        return new PlotResponseStats(maxValue, values, average, dataSeriesStats);
-    }
 }