andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

Merge branch 'master' of https://bitbucket.com/andreashuber/performance

Browse Source 
# Conflicts:
#	build.gradle
This commit is contained in:
Andreas Huber
2020-08-11 19:48:09 +02:00
parent 275b6945cc 668f86323e
commit 5b55badfea
12 changed files with 83 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
build.gradle
View File

@@ -31,7 +31,11 @@ ext {
lib_commons_collections4 = 'org.apache.commons:commons-collections4:4.4'
lib_commons_lang3 = 'org.apache.commons:commons-lang3:3.10'
<<<<<<< HEAD
lib_jackson_databind = 'com.fasterxml.jackson.core:jackson-databind:2.11.2'
=======
lib_jackson_databind = 'com.fasterxml.jackson.core:jackson-databind:2.11.0'
>>>>>>> 668f86323eb4efa2f2c5a016e8785958517af07b
lib_log4j2_core = "org.apache.logging.log4j:log4j-core:${version_log4j2}"
lib_log4j2_slf4j_impl = "org.apache.logging.log4j:log4j-slf4j-impl:${version_log4j2}"
@@ -124,5 +128,9 @@ subprojects {
}
wrapper {
<<<<<<< HEAD
gradleVersion = '6.6'
=======
gradleVersion = '6.4'
>>>>>>> 668f86323eb4efa2f2c5a016e8785958517af07b
}

13
pdb-js/src/app/app.component.html
View File

@@ -1,21 +1,8 @@
<style>
:host {
}
</style>
<div id="main-toolbar">
<a href="/" title="open main page"><img src="assets/img/home.svg" class="icon-small" aria-hidden="false" aria-label="go to main page" /></a>
<a href="/vis" title="open visualization page"><img src="assets/img/scatter-chart2.svg" class="icon-small" aria-hidden="false" aria-label="go to visualization page" /></a>
<!--a href="/upload" title="upload data"><img src="assets/img/upload.svg" class="icon-small" aria-hidden="false" aria-label="upload data" /></a-->
<!--
<a href="/help" title="open help page" class="right"><img src="assets/img/question-mark-round.svg" class="icon-small" aria-hidden="false" aria-label="help" /></a>
-->
</div>
<router-outlet></router-outlet>

69
pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java
View File

@@ -1,5 +1,6 @@
package org.lucares.pdbui;
import java.nio.charset.StandardCharsets;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
@@ -38,51 +39,51 @@ public final class CsvReaderSettings {
}
public static final class ColumnDefinitions {
Map<String, ColumnDefinition> columnDefinitions = new HashMap<>();
Map<String, ColumnDefinition> columns = new HashMap<>();
public Map<String, ColumnDefinition> getColumnDefinitions() {
return columnDefinitions;
public Map<String, ColumnDefinition> getColumns() {
return columns;
}
public void setColumnDefinitions(final Map<String, ColumnDefinition> columnDefinitions) {
this.columnDefinitions = columnDefinitions;
public void setColumns(final Map<String, ColumnDefinition> columnDefinitions) {
this.columns = columnDefinitions;
}
public void ignoreColumn(final String csvColumnHeader) {
columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columnDefinitions.get(csvColumnHeader).setIgnore(true);
columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columns.get(csvColumnHeader).setIgnore(true);
}
public void rename(final String csvColumnHeader, final String renameTo) {
columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columnDefinitions.get(csvColumnHeader).setRenameTo(renameTo);
columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columns.get(csvColumnHeader).setRenameTo(renameTo);
}
public void postProcess(final String csvColumnHeader, final EnumSet<PostProcessors> postProcessors) {
columnDefinitions.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columnDefinitions.get(csvColumnHeader).setPostProcessors(postProcessors);
columns.putIfAbsent(csvColumnHeader, new ColumnDefinition());
columns.get(csvColumnHeader).setPostProcessors(postProcessors);
}
public boolean isIgnoredColumn(final String csvColumnHeader) {
return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).isIgnore();
return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).isIgnore();
}
public String getRenameTo(final String csvColumnHeader) {
return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).getRenameTo();
return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).getRenameTo();
}
public EnumSet<PostProcessors> getPostProcessors(final String csvColumnHeader) {
return columnDefinitions.getOrDefault(csvColumnHeader, new ColumnDefinition()).getPostProcessors();
return columns.getOrDefault(csvColumnHeader, new ColumnDefinition()).getPostProcessors();
}
@Override
public String toString() {
final StringBuilder result = new StringBuilder();
for (final String col : columnDefinitions.keySet()) {
for (final String col : columns.keySet()) {
result.append(col);
result.append(":");
result.append(columnDefinitions.get(col));
result.append(columns.get(col));
result.append("\n");
}
@@ -145,7 +146,7 @@ public final class CsvReaderSettings {
}
}
private byte separator;
private String separator;
private ColumnDefinitions columnDefinitions = new ColumnDefinitions();
@@ -155,13 +156,13 @@ public final class CsvReaderSettings {
private String valueColumn;
private byte comment = '#';
private String comment = "#";
public CsvReaderSettings() {
this("@timestamp", "duration", (byte) ',', new ColumnDefinitions());
this("@timestamp", "duration", ",", new ColumnDefinitions());
}
private CsvReaderSettings(final String timeColumn, final String valueColumn, final byte separator,
private CsvReaderSettings(final String timeColumn, final String valueColumn, final String separator,
final ColumnDefinitions columnDefinitions) {
this.timeColumn = timeColumn;
@@ -170,13 +171,13 @@ public final class CsvReaderSettings {
this.columnDefinitions = columnDefinitions;
}
public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final char separator,
public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final String separator,
final ColumnDefinitions columnDefinitions) {
Preconditions.checkTrue(separator == (byte) separator,
Preconditions.checkTrue(separator.getBytes(StandardCharsets.UTF_8).length == 1,
"Only separators that fulfill separator == (byte)separator are supported. "
+ "This restriction is because the parsing algorithm skips the overhead of "
+ "translating bytes to characters.");
return new CsvReaderSettings(timeColumn, valueColumn, (byte) separator, columnDefinitions);
return new CsvReaderSettings(timeColumn, valueColumn, separator, columnDefinitions);
}
public String getTimeColumn() {
@@ -195,22 +196,36 @@ public final class CsvReaderSettings {
this.valueColumn = valueColumn;
}
public byte getSeparator() {
public String getSeparator() {
return separator;
}
public void setSeparator(final byte separator) {
public void setSeparator(final String separator) {
this.separator = separator;
}
public byte getComment() {
public byte separatorByte() {
final byte[] bytes = separator.getBytes(StandardCharsets.UTF_8);
Preconditions.checkEqual(bytes.length, 1,
"separator must be a character that is represented as a single byte in UTF-8");
return bytes[0];
}
public String getComment() {
return comment;
}
public void setComment(final byte comment) {
public void setComment(final String comment) {
this.comment = comment;
}
public byte commentByte() {
final byte[] bytes = comment.getBytes(StandardCharsets.UTF_8);
Preconditions.checkEqual(bytes.length, 1,
"comment must be a character that is represented as a single byte in UTF-8");
return bytes[0];
}
public void putAdditionalTag(final String field, final String value) {
additionalTags.put(field, value);
}

4
pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java
View File

@@ -46,8 +46,8 @@ class CsvToEntryTransformer {
Entries entries = new Entries(chunksize);
final byte newline = '\n';
final byte separator = settings.getSeparator();
final byte comment = settings.getComment();
final byte separator = settings.separatorByte();
final byte comment = settings.commentByte();
final byte[] line = new byte[64 * 1024]; // max line length
int offsetInLine = 0;
int offsetInBuffer = 0;

4
pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java
View File

@@ -54,7 +54,7 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean {
for (final MultipartFile file : files) {
final Path tmpFile = tmpDir.resolve(UUID.randomUUID().toString());
tmpFiles.add(tmpFile);
LOGGER.info("writing uploaded file to {}", tmpFile);
LOGGER.debug("writing uploaded file to {}", tmpFile);
file.transferTo(tmpFile);
}
} catch (RuntimeException | IOException e) {
@@ -71,7 +71,7 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean {
csvToEntryTransformer.readCSV(in);
}
LOGGER.info("delete uploaded file {}", tmpFile);
LOGGER.debug("delete uploaded file {}", tmpFile);
Files.delete(tmpFile);
} catch (final Exception e) {

2
pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java
View File

@@ -68,7 +68,7 @@ public final class IngestionHandler implements Callable<Void> {
} else {
in.reset();
final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue,
CsvReaderSettings.create("@timestamp", "duration", ',', new ColumnDefinitions()));
CsvReaderSettings.create("@timestamp", "duration", ",", new ColumnDefinitions()));
csvTransformer.readCSV(in);
}
}

8
pdb-ui/src/main/java/org/lucares/pdbui/date/FastISODateParser.java
View File

@@ -192,7 +192,11 @@ public class FastISODateParser {
final long nanos = nanosAndCharsRead[0];
final int offsetTimezone = beginIndex + 19 + nanosAndCharsRead[1];
final long zoneOffsetMillis = date[offsetTimezone] == 'Z' ? 0 : parseZoneToMillis(date, offsetTimezone);
final byte firstTimeZoneChar = date[offsetTimezone];
final boolean isNumericTimeZone = firstTimeZoneChar >= '0' && firstTimeZoneChar <= '9'
|| (firstTimeZoneChar == '-' || firstTimeZoneChar == '+');
final long zoneOffsetMillis = firstTimeZoneChar == 'Z' || !isNumericTimeZone ? 0
: parseZoneToMillis(date, offsetTimezone);
final int epochMilliMonthOffsetKey = (int) (year * 12 + month - 1);
final long epochMilliMonthOffset;
@@ -261,7 +265,7 @@ public class FastISODateParser {
while (i < date.length) {
final byte c = date[i];
i++;
if (c == '.') {
if (c == '.' || c == ',') {
continue;
}
if (c < '0' || c > '9') {

6
pdb-ui/src/main/resources/application-testing.properties
View File

@@ -1,6 +1,8 @@
#db.base=D:/ws/pdb/dataNew
db.base=C:/Temp/pdb/testing
db.base=D:/ws/pdb/databases/test
server.port=17333
gnuplot.home=D:/ws/pdb/gnuplot-5.2
cache.images.duration.seconds=86400
defaults.groupBy=pod,method,metric
defaults.splitBy=method
defaults.query.examples=pod=vapfinra01 and method=ViewService.findFieldView,ViewService.findFieldViewGroup;pod=vappilby01 and method=ReviewInContextController.index;pod=vapnyse001 and method=ReviewInContextController.index,ReviewController.index
mode.production=false

4
pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java
View File

@@ -50,7 +50,7 @@ public class CsvToEntryTransformerTest {
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',',
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ",",
new ColumnDefinitions());
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
@@ -92,7 +92,7 @@ public class CsvToEntryTransformerTest {
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final ColumnDefinitions columnDefinitions = new ColumnDefinitions();
columnDefinitions.ignoreColumn("ignoredColumn");
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',',
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ",",
columnDefinitions);
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));

2
pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java
View File

@@ -66,7 +66,7 @@ public class PdbControllerTest {
final ColumnDefinitions columnDefinitions = new ColumnDefinitions();
columnDefinitions.ignoreColumn(ignoredColumn);
columnDefinitions.postProcess("tag", EnumSet.of(PostProcessors.LOWER_CASE));
final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, valueColumn, ',', columnDefinitions);
final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, valueColumn, ",", columnDefinitions);
settings.putAdditionalTag(additionalColumn, additionalValue);
uploadCsv(settings, csv);
{

11
pdb-ui/src/test/java/org/lucares/pdbui/date/FastISODateParserTest.java
View File

@@ -12,10 +12,10 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.api.Assertions;
public class FastISODateParserTest {
@@ -44,7 +44,8 @@ public class FastISODateParserTest {
Arguments.of("2018-11-18T14:55:49.Z"), //
Arguments.of("2018-11-18T14:55:49.-01:23"), //
Arguments.of("2018-11-18T14:55:49Z"), //
Arguments.of("2018-11-18T14:55:49-01:23") //
Arguments.of("2018-11-18T14:55:49-01:23"), //
Arguments.of("2018-11-18 14:55:49,123") //
);
}
@@ -187,7 +188,7 @@ public class FastISODateParserTest {
Assertions.assertEquals(expectedEpochMilli, actualEpochMilli, "date: " + date);
}
public static void main(final String[] args) throws IOException, InterruptedException {
public static void main2(final String[] args) throws IOException, InterruptedException {
final Path path = Path.of("/home/andi/ws/performanceDb/data/production/dates.csv");
final List<byte[]> dates = new ArrayList<>();
@@ -228,4 +229,8 @@ public class FastISODateParserTest {
System.out.println("duration: " + millis + "ms ; speed: " + datesPerSecond + " dates/s");
}
}
public static void main(final String[] args) {
System.out.println(new FastISODateParser().parse("2018-11-18 14:55:49,123"));
}
}

7
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -137,8 +137,10 @@ public class PerformanceDb implements AutoCloseable {
}
if (entries.isForceFlush()) {
LOGGER.info("flush triggered via entries.isForceFlush()");
LOGGER.debug("flush triggered via entries.isForceFlush()");
final long start = System.nanoTime();
dataStore.flush();
LOGGER.debug("flush duration: {}ms", (System.nanoTime() - start) / 1_000_000.0);
entries.notifyFlushed();
}
}
@@ -147,8 +149,9 @@ public class PerformanceDb implements AutoCloseable {
throw new WriteException(e);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.info("Thread was interrupted. Aborting exectution.");
LOGGER.info("Thread was interrupted. Aborting execution.");
} finally {
LOGGER.info("flush after inserting all data");
dataStore.flush();
}
}