andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

make it possible to ignore columns using the csv ingestor

Browse Source
This commit is contained in:
Andreas Huber
2019-07-04 09:51:33 +02:00
parent 3a39f66e22
commit 2cb81e5acd
4 changed files with 45 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pdb-ui/src/main/java/org/lucares/pdbui/TcpIngestor.java
View File

@@ -58,6 +58,11 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
public final static class Handler implements Callable<Void> {
/**
* Column header names starting with "-" will be ignored.
*/
static final String COLUM_IGNORE_PREFIX = "-";
private static final int IGNORE_COLUMN = 0;
final Socket clientSocket;
private final ArrayBlockingQueue<Entries> queue;
@@ -199,15 +204,21 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
for (int i = 0; i < size; i++) {
final int separatorPosition = separatorPositions.get(i);
final int value = Tags.STRING_COMPRESSOR.put(line, lastSeparatorPosition + 1, separatorPosition);
final int compressedString = Tags.STRING_COMPRESSOR.put(line, lastSeparatorPosition + 1,
separatorPosition);
final String columnName = Tags.STRING_COMPRESSOR.get(compressedString);
columns[i] = value;
columns[i] = ignoreColum(columnName) ? IGNORE_COLUMN : compressedString;
lastSeparatorPosition = separatorPosition;
}
return columns;
}
private boolean ignoreColum(final String columnName) {
return columnName.startsWith(COLUM_IGNORE_PREFIX);
}
private static Entry handleCsvLine(final int[] columns, final byte[] line, final int bytesInLine,
final IntList separatorPositions, final int keyTimestamp, final int keyDuration,
final FastISODateParser dateParser) {
@@ -224,7 +235,9 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
final int separatorPosition = separatorPositions.get(i);
final int key = columns[i];
if (key == keyTimestamp) {
if (key == IGNORE_COLUMN) {
// this column's value will not be ingested
} else if (key == keyTimestamp) {
epochMilli = dateParser.parseAsEpochMilli(line, lastSeparatorPosition + 1);
} else if (key == keyDuration) {
duration = parseLong(line, lastSeparatorPosition + 1);