andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

ignore comment lines in CSV files

Browse Source
This commit is contained in:
Andreas Huber
2019-12-14 08:11:12 +01:00
parent 4e554bfa85
commit 204c258980
3 changed files with 18 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java
View File

@@ -1,6 +1,7 @@
package org.lucares.pdbui; package org.lucares.pdbui;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@@ -18,9 +19,10 @@ public class CsvReaderSettings {
private String timeColumn; private String timeColumn;
private byte comment = '#';
public CsvReaderSettings() { public CsvReaderSettings() {
separator = ','; this("@timestamp", (byte) ',', Collections.emptyList());
timeColumn = "@timestamp";
} }
private CsvReaderSettings(final String timeColumn, final byte separator, final Collection<String> ignoreColumns) { private CsvReaderSettings(final String timeColumn, final byte separator, final Collection<String> ignoreColumns) {
@@ -65,6 +67,14 @@ public class CsvReaderSettings {
this.separator = separator; this.separator = separator;
} }
public byte getComment() {
return comment;
}
public void setComment(final byte comment) {
this.comment = comment;
}
public Set<String> getIgnoreColumnNames() { public Set<String> getIgnoreColumnNames() {
return ignoreColumnNames; return ignoreColumnNames;
} }

5
pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java
View File

@@ -38,6 +38,7 @@ class CsvToEntryTransformer {
final byte newline = '\n'; final byte newline = '\n';
final byte separator = settings.getSeparator(); final byte separator = settings.getSeparator();
final byte comment = settings.getComment();
final byte[] line = new byte[64 * 1024]; // max line length final byte[] line = new byte[64 * 1024]; // max line length
int offsetInLine = 0; int offsetInLine = 0;
int offsetInBuffer = 0; int offsetInBuffer = 0;
@@ -64,7 +65,9 @@ class CsvToEntryTransformer {
bytesInLine = offsetInLine + length; bytesInLine = offsetInLine + length;
separatorPositions.add(offsetInLine + i - offsetInBuffer); separatorPositions.add(offsetInLine + i - offsetInBuffer);
if (columns != null) { if (line[0] == comment) {
// ignore
} else if (columns != null) {
final Entry entry = handleCsvLine(columns, line, bytesInLine, separatorPositions, keyTimestamp, final Entry entry = handleCsvLine(columns, line, bytesInLine, separatorPositions, keyTimestamp,
keyDuration, dateParser, additionalTags); keyDuration, dateParser, additionalTags);

3
pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java
View File

@@ -54,7 +54,8 @@ public class PdbControllerTest {
final OffsetDateTime dateA = OffsetDateTime.now(); final OffsetDateTime dateA = OffsetDateTime.now();
final OffsetDateTime dateB = OffsetDateTime.now(); final OffsetDateTime dateB = OffsetDateTime.now();
final String csv = timeColumn + ",duration,tag," + ignoredColumn + "\n"// final String csv = "# first line is a comment\n"//
+ timeColumn + ",duration,tag," + ignoredColumn + "\n"//
+ dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue,ignoredValue\n"// + dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue,ignoredValue\n"//
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue,ignoredValue\n"; + dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue,ignoredValue\n";