Merge branch 'master' of https://bitbucket.com/andreashuber/performance

# Conflicts:
#	build.gradle

build.gradle

@@ -31,7 +31,11 @@ ext {
     
     lib_commons_collections4 = 'org.apache.commons:commons-collections4:4.4'
     lib_commons_lang3 = 'org.apache.commons:commons-lang3:3.10'
+<<<<<<< HEAD
     lib_jackson_databind = 'com.fasterxml.jackson.core:jackson-databind:2.11.2'
+=======
+    lib_jackson_databind = 'com.fasterxml.jackson.core:jackson-databind:2.11.0'
+>>>>>>> 668f86323eb4efa2f2c5a016e8785958517af07b
     
     lib_log4j2_core = "org.apache.logging.log4j:log4j-core:${version_log4j2}"
     lib_log4j2_slf4j_impl = "org.apache.logging.log4j:log4j-slf4j-impl:${version_log4j2}"
@@ -124,5 +128,9 @@ subprojects {
 }
 
 wrapper {
+<<<<<<< HEAD
     gradleVersion = '6.6'
+=======
+    gradleVersion = '6.4'
+>>>>>>> 668f86323eb4efa2f2c5a016e8785958517af07b
 }

pdb-js/src/app/app.component.html

@@ -1,21 +1,8 @@
 
 
-<style>
-  :host {
-  }
-  
-  
-  
-</style>
-
-
 <div id="main-toolbar">
-  <a href="/" title="open main page"><img src="assets/img/home.svg" class="icon-small" aria-hidden="false" aria-label="go to main page" /></a> 
+  <a href="/" title="open main page"><img src="assets/img/home.svg" class="icon-small" aria-hidden="false" aria-label="go to main page" /></a>
   <a href="/vis" title="open visualization page"><img src="assets/img/scatter-chart2.svg" class="icon-small" aria-hidden="false" aria-label="go to visualization page" /></a>
-  <!--a href="/upload" title="upload data"><img src="assets/img/upload.svg" class="icon-small" aria-hidden="false" aria-label="upload data" /></a-->
-  <!--
-  <a href="/help" title="open help page" class="right"><img src="assets/img/question-mark-round.svg" class="icon-small" aria-hidden="false" aria-label="help" /></a>
-  -->
 </div>
 
-<router-outlet></router-outlet>
+<router-outlet></router-outlet>

pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java

@@ -1,5 +1,6 @@
 package org.lucares.pdbui;
 
+import java.nio.charset.StandardCharsets;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -38,51 +39,51 @@ public final class CsvReaderSettings {
     }
 
     public static final class ColumnDefinitions {
-        Map<String, ColumnDefinition> columnDefinitions = new HashMap<>();
+        Map<String, ColumnDefinition> columns = new HashMap<>();
 
-        public Map<String, ColumnDefinition> getColumnDefinitions() {
+        public Map<String, ColumnDefinition> getColumns() {
-            return columnDefinitions;
+            return columns;
         }
 
-        public void setColumnDefinitions(final Map<String, ColumnDefinition> columnDefinitions) {
+        public void setColumns(final Map<String, ColumnDefinition> columnDefinitions) {
-            this.columnDefinitions = columnDefinitions;
+            this.columns = columnDefinitions;
         }
 
         public void ignoreColumn(final String csvColumnHeader) {
-            columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
+            columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
-            columnDefinitions.get(csvColumnHeader).setIgnore(true);
+            columns.get(csvColumnHeader).setIgnore(true);
         }
 
         public void rename(final String csvColumnHeader, final String renameTo) {
-            columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
+            columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
-            columnDefinitions.get(csvColumnHeader).setRenameTo(renameTo);
+            columns.get(csvColumnHeader).setRenameTo(renameTo);
         }
 
         public void postProcess(final String csvColumnHeader, final EnumSet<PostProcessors> postProcessors) {
-            columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
+            columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
-            columnDefinitions.get(csvColumnHeader).setPostProcessors(postProcessors);
+            columns.get(csvColumnHeader).setPostProcessors(postProcessors);
         }
 
         public boolean isIgnoredColumn(final String csvColumnHeader) {
-            return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).isIgnore();
+            return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).isIgnore();
         }
 
         public String getRenameTo(final String csvColumnHeader) {
-            return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).getRenameTo();
+            return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).getRenameTo();
         }
 
         public EnumSet<PostProcessors> getPostProcessors(final String csvColumnHeader) {
-            return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).getPostProcessors();
+            return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).getPostProcessors();
         }
 
         @Override
         public String toString() {
             final StringBuilder result = new StringBuilder();
 
-            for (final String col : columnDefinitions.keySet()) {
+            for (final String col : columns.keySet()) {
                 result.append(col);
                 result.append(":");
-                result.append(columnDefinitions.get(col));
+                result.append(columns.get(col));
                 result.append("\n");
             }
 
@@ -145,7 +146,7 @@ public final class CsvReaderSettings {
         }
     }
 
-    private byte separator;
+    private String separator;
 
     private ColumnDefinitions columnDefinitions = new ColumnDefinitions();
 
@@ -155,13 +156,13 @@ public final class CsvReaderSettings {
 
     private String valueColumn;
 
-    private byte comment = '#';
+    private String comment = "#";
 
     public CsvReaderSettings() {
-        this("@timestamp", "duration", (byte) ',', new ColumnDefinitions());
+        this("@timestamp", "duration", ",", new ColumnDefinitions());
     }
 
-    private CsvReaderSettings(final String timeColumn, final String valueColumn, final byte separator,
+    private CsvReaderSettings(final String timeColumn, final String valueColumn, final String separator,
             final ColumnDefinitions columnDefinitions) {
 
         this.timeColumn = timeColumn;
@@ -170,13 +171,13 @@ public final class CsvReaderSettings {
         this.columnDefinitions = columnDefinitions;
     }
 
-    public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final char separator,
+    public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final String separator,
             final ColumnDefinitions columnDefinitions) {
-        Preconditions.checkTrue(separator == (byte) separator,
+        Preconditions.checkTrue(separator.getBytes(StandardCharsets.UTF_8).length == 1,
                 "Only separators that fulfill separator == (byte)separator are supported. "
                         + "This restriction is because the parsing algorithm skips the overhead of "
                         + "translating bytes to characters.");
-        return new CsvReaderSettings(timeColumn, valueColumn, (byte) separator, columnDefinitions);
+        return new CsvReaderSettings(timeColumn, valueColumn, separator, columnDefinitions);
     }
 
     public String getTimeColumn() {
@@ -195,22 +196,36 @@ public final class CsvReaderSettings {
         this.valueColumn = valueColumn;
     }
 
-    public byte getSeparator() {
+    public String getSeparator() {
         return separator;
     }
 
-    public void setSeparator(final byte separator) {
+    public void setSeparator(final String separator) {
         this.separator = separator;
     }
 
-    public byte getComment() {
+    public byte separatorByte() {
+        final byte[] bytes = separator.getBytes(StandardCharsets.UTF_8);
+        Preconditions.checkEqual(bytes.length, 1,
+                "separator must be a character that is represented as a single byte in UTF-8");
+        return bytes[0];
+    }
+
+    public String getComment() {
         return comment;
     }
 
-    public void setComment(final byte comment) {
+    public void setComment(final String comment) {
         this.comment = comment;
     }
 
+    public byte commentByte() {
+        final byte[] bytes = comment.getBytes(StandardCharsets.UTF_8);
+        Preconditions.checkEqual(bytes.length, 1,
+                "comment must be a character that is represented as a single byte in UTF-8");
+        return bytes[0];
+    }
+
     public void putAdditionalTag(final String field, final String value) {
         additionalTags.put(field, value);
     }

pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java

@@ -46,8 +46,8 @@ class CsvToEntryTransformer {
         Entries entries = new Entries(chunksize);
 
         final byte newline = '\n';
-        final byte separator = settings.getSeparator();
+        final byte separator = settings.separatorByte();
-        final byte comment = settings.getComment();
+        final byte comment = settings.commentByte();
         final byte[] line = new byte[64 * 1024]; // max line length
         int offsetInLine = 0;
         int offsetInBuffer = 0;

pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java

@@ -54,7 +54,7 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean {
             for (final MultipartFile file : files) {
                 final Path tmpFile = tmpDir.resolve(UUID.randomUUID().toString());
                 tmpFiles.add(tmpFile);
-                LOGGER.info("writing uploaded file to {}", tmpFile);
+                LOGGER.debug("writing uploaded file to {}", tmpFile);
                 file.transferTo(tmpFile);
             }
         } catch (RuntimeException | IOException e) {
@@ -71,7 +71,7 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean {
                         csvToEntryTransformer.readCSV(in);
                     }
 
-                    LOGGER.info("delete uploaded file {}", tmpFile);
+                    LOGGER.debug("delete uploaded file {}", tmpFile);
                     Files.delete(tmpFile);
 
                 } catch (final Exception e) {

pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java

@@ -68,7 +68,7 @@ public final class IngestionHandler implements Callable<Void> {
         } else {
             in.reset();
             final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue,
-                    CsvReaderSettings.create("@timestamp", "duration", ',', new ColumnDefinitions()));
+                    CsvReaderSettings.create("@timestamp", "duration", ",", new ColumnDefinitions()));
             csvTransformer.readCSV(in);
         }
     }

pdb-ui/src/main/java/org/lucares/pdbui/date/FastISODateParser.java

@@ -192,7 +192,11 @@ public class FastISODateParser {
             final long nanos = nanosAndCharsRead[0];
             final int offsetTimezone = beginIndex + 19 + nanosAndCharsRead[1];
 
-            final long zoneOffsetMillis = date[offsetTimezone] == 'Z' ? 0 : parseZoneToMillis(date, offsetTimezone);
+            final byte firstTimeZoneChar = date[offsetTimezone];
+            final boolean isNumericTimeZone = firstTimeZoneChar >= '0' && firstTimeZoneChar <= '9'
+                    || (firstTimeZoneChar == '-' || firstTimeZoneChar == '+');
+            final long zoneOffsetMillis = firstTimeZoneChar == 'Z' || !isNumericTimeZone ? 0
+                    : parseZoneToMillis(date, offsetTimezone);
 
             final int epochMilliMonthOffsetKey = (int) (year * 12 + month - 1);
             final long epochMilliMonthOffset;
@@ -261,7 +265,7 @@ public class FastISODateParser {
         while (i < date.length) {
             final byte c = date[i];
             i++;
-            if (c == '.') {
+            if (c == '.' || c == ',') {
                 continue;
             }
             if (c < '0' || c > '9') {

pdb-ui/src/main/resources/application-testing.properties

@@ -1,6 +1,8 @@
-#db.base=D:/ws/pdb/dataNew
+db.base=D:/ws/pdb/databases/test
-db.base=C:/Temp/pdb/testing
 server.port=17333
 gnuplot.home=D:/ws/pdb/gnuplot-5.2
 cache.images.duration.seconds=86400
+defaults.groupBy=pod,method,metric
+defaults.splitBy=method
+defaults.query.examples=pod=vapfinra01 and method=ViewService.findFieldView,ViewService.findFieldViewGroup;pod=vappilby01 and method=ReviewInContextController.index;pod=vapnyse001 and method=ReviewInContextController.index,ReviewController.index
 mode.production=false

pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java

@@ -50,7 +50,7 @@ public class CsvToEntryTransformerTest {
                     + dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
 
             final ArrayBlockingQueue<Entries> queue = db.getQueue();
-            final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',',
+            final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ",",
                     new ColumnDefinitions());
             final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
             csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
@@ -92,7 +92,7 @@ public class CsvToEntryTransformerTest {
             final ArrayBlockingQueue<Entries> queue = db.getQueue();
             final ColumnDefinitions columnDefinitions = new ColumnDefinitions();
             columnDefinitions.ignoreColumn("ignoredColumn");
-            final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',',
+            final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ",",
                     columnDefinitions);
             final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
             csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));

pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java

@@ -66,7 +66,7 @@ public class PdbControllerTest {
         final ColumnDefinitions columnDefinitions = new ColumnDefinitions();
         columnDefinitions.ignoreColumn(ignoredColumn);
         columnDefinitions.postProcess("tag", EnumSet.of(PostProcessors.LOWER_CASE));
-        final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, valueColumn, ',', columnDefinitions);
+        final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, valueColumn, ",", columnDefinitions);
         settings.putAdditionalTag(additionalColumn, additionalValue);
         uploadCsv(settings, csv);
         {

pdb-ui/src/test/java/org/lucares/pdbui/date/FastISODateParserTest.java

@@ -12,10 +12,10 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
-import org.junit.jupiter.api.Assertions;
 
 public class FastISODateParserTest {
 
@@ -44,7 +44,8 @@ public class FastISODateParserTest {
                 Arguments.of("2018-11-18T14:55:49.Z"), //
                 Arguments.of("2018-11-18T14:55:49.-01:23"), //
                 Arguments.of("2018-11-18T14:55:49Z"), //
-                Arguments.of("2018-11-18T14:55:49-01:23") //
+                Arguments.of("2018-11-18T14:55:49-01:23"), //
+                Arguments.of("2018-11-18 14:55:49,123") //
         );
 
     }
@@ -187,7 +188,7 @@ public class FastISODateParserTest {
         Assertions.assertEquals(expectedEpochMilli, actualEpochMilli, "date: " + date);
     }
 
-    public static void main(final String[] args) throws IOException, InterruptedException {
+    public static void main2(final String[] args) throws IOException, InterruptedException {
         final Path path = Path.of("/home/andi/ws/performanceDb/data/production/dates.csv");
 
         final List<byte[]> dates = new ArrayList<>();
@@ -228,4 +229,8 @@ public class FastISODateParserTest {
             System.out.println("duration: " + millis + "ms ; speed: " + datesPerSecond + " dates/s");
         }
     }
+
+    public static void main(final String[] args) {
+        System.out.println(new FastISODateParser().parse("2018-11-18 14:55:49,123"));
+    }
 }

performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java

@@ -137,8 +137,10 @@ public class PerformanceDb implements AutoCloseable {
                 }
 
                 if (entries.isForceFlush()) {
-                    LOGGER.info("flush triggered via entries.isForceFlush()");
+                    LOGGER.debug("flush triggered via entries.isForceFlush()");
+                    final long start = System.nanoTime();
                     dataStore.flush();
+                    LOGGER.debug("flush duration: {}ms", (System.nanoTime() - start) / 1_000_000.0);
                     entries.notifyFlushed();
                 }
             }
@@ -147,8 +149,9 @@ public class PerformanceDb implements AutoCloseable {
             throw new WriteException(e);
         } catch (final InterruptedException e) {
             Thread.currentThread().interrupt();
-            LOGGER.info("Thread was interrupted. Aborting exectution.");
+            LOGGER.info("Thread was interrupted. Aborting execution.");
         } finally {
+            LOGGER.info("flush after inserting all data");
             dataStore.flush();
         }
     }