andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

define name of value column

Browse Source 
The name of the column containing the value is now part of the
csv settings. This allows us to use different names than "duration".
This commit is contained in:
Andreas Huber
2019-12-14 08:34:33 +01:00
parent 204c258980
commit 7737d45887
5 changed files with 27 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java
View File

@@ -19,36 +19,40 @@ public class CsvReaderSettings {
private String timeColumn;
private String valueColumn;
private byte comment = '#';
public CsvReaderSettings() {
this("@timestamp", (byte) ',', Collections.emptyList());
this("@timestamp", "duration", (byte) ',', Collections.emptyList());
}
private CsvReaderSettings(final String timeColumn, final byte separator, final Collection<String> ignoreColumns) {
private CsvReaderSettings(final String timeColumn, final String valueColumn, final byte separator,
final Collection<String> ignoreColumns) {
this.timeColumn = timeColumn;
this.valueColumn = valueColumn;
this.separator = separator;
this.ignoreColumnNames.addAll(ignoreColumns);
}
public static CsvReaderSettings create(final String timeColumn, final byte separator,
public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final byte separator,
final String... ignoreColumnNames) {
return new CsvReaderSettings(timeColumn, separator, List.of(ignoreColumnNames));
return new CsvReaderSettings(timeColumn, valueColumn, separator, List.of(ignoreColumnNames));
}
public static CsvReaderSettings create(final String timeColumn, final char separator,
public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final char separator,
final String... ignoreColumnNames) {
return CsvReaderSettings.create(timeColumn, separator, List.of(ignoreColumnNames));
return CsvReaderSettings.create(timeColumn, valueColumn, separator, List.of(ignoreColumnNames));
}
public static CsvReaderSettings create(final String timeColumn, final char separator,
public static CsvReaderSettings create(final String timeColumn, final String valueColumn, final char separator,
final Collection<String> ignoreColumnNames) {
Preconditions.checkTrue(separator == (byte) separator,
"Only separators that fulfill separator == (byte)separator are supported. "
+ "This restriction is because the parsing algorithm skips the overhead of "
+ "translating bytes to characters.");
return new CsvReaderSettings(timeColumn, (byte) separator, ignoreColumnNames);
return new CsvReaderSettings(timeColumn, valueColumn, (byte) separator, ignoreColumnNames);
}
public String getTimeColumn() {
@@ -59,6 +63,14 @@ public class CsvReaderSettings {
this.timeColumn = timeColumn;
}
public String getValueColumn() {
return valueColumn;
}
public void setValueColumn(final String valueColumn) {
this.valueColumn = valueColumn;
}
public byte getSeparator() {
return separator;
}

2
pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java
View File

@@ -50,7 +50,7 @@ class CsvToEntryTransformer {
int[] columns = null;
final byte[] buffer = new byte[4096 * 16];
final int keyTimestamp = Tags.STRING_COMPRESSOR.put(settings.getTimeColumn());
final int keyDuration = Tags.STRING_COMPRESSOR.put("duration");
final int keyDuration = Tags.STRING_COMPRESSOR.put(settings.getValueColumn());
final FastISODateParser dateParser = new FastISODateParser();
final Tags additionalTags = initAditionalTags();

2
pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java
View File

@@ -67,7 +67,7 @@ public final class IngestionHandler implements Callable<Void> {
} else {
in.reset();
final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue,
CsvReaderSettings.create("@timestamp", ','));
CsvReaderSettings.create("@timestamp", "duration", ','));
csvTransformer.readCSV(in);
}
}

4
pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java
View File

@@ -49,7 +49,7 @@ public class CsvToEntryTransformerTest {
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", ',');
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',');
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
queue.put(Entries.POISON);
@@ -88,7 +88,7 @@ public class CsvToEntryTransformerTest {
+ "2000-01-01T00:00:00.001Z,2,ignoreValue,ignoreValue,tagValue\n";
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", ',', "ignoredColumn");
final CsvReaderSettings settings = CsvReaderSettings.create("@timestamp", "duration", ',', "ignoredColumn");
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
queue.put(Entries.POISON);

5
pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java
View File

@@ -51,15 +51,16 @@ public class PdbControllerTest {
final String additionalValue = "additionalValue";
final String ignoredColumn = "ignoredColumn";
final String timeColumn = "time";
final String valueColumn = "value";
final OffsetDateTime dateA = OffsetDateTime.now();
final OffsetDateTime dateB = OffsetDateTime.now();
final String csv = "# first line is a comment\n"//
+ timeColumn + ",duration,tag," + ignoredColumn + "\n"//
+ timeColumn + "," + valueColumn + ",tag," + ignoredColumn + "\n"//
+ dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue,ignoredValue\n"//
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue,ignoredValue\n";
final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, ',', ignoredColumn);
final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, valueColumn, ',', ignoredColumn);
settings.putAdditionalTag(additionalColumn, additionalValue);
uploadCsv(settings, csv);
{