make time column configurable

Adding the header of the time column to the
CsvReaderSettings. This is necessary to parse
arbitrary CSV files.
This commit is contained in:
2019-12-09 18:59:48 +01:00
parent d383134c42
commit 394e16ad27
5 changed files with 34 additions and 18 deletions

View File

@@ -12,30 +12,45 @@ public class CsvReaderSettings {
private Set<String> ignoreColumnNames = new HashSet<String>();
private String timeColumn;
public CsvReaderSettings() {
separator = ',';
timeColumn = "@timestamp";
}
private CsvReaderSettings(final byte separator, final Collection<String> ignoreColumns) {
private CsvReaderSettings(final String timeColumn, final byte separator, final Collection<String> ignoreColumns) {
this.timeColumn = timeColumn;
this.separator = separator;
this.ignoreColumnNames.addAll(ignoreColumns);
}
public static CsvReaderSettings create(final byte separator, final String... ignoreColumnNames) {
return new CsvReaderSettings(separator, List.of(ignoreColumnNames));
public static CsvReaderSettings create(final String timeColumn, final byte separator,
final String... ignoreColumnNames) {
return new CsvReaderSettings(timeColumn, separator, List.of(ignoreColumnNames));
}
public static CsvReaderSettings create(final char separator, final String... ignoreColumnNames) {
return CsvReaderSettings.create(separator, List.of(ignoreColumnNames));
public static CsvReaderSettings create(final String timeColumn, final char separator,
final String... ignoreColumnNames) {
return CsvReaderSettings.create(timeColumn, separator, List.of(ignoreColumnNames));
}
public static CsvReaderSettings create(final char separator, final Collection<String> ignoreColumnNames) {
public static CsvReaderSettings create(final String timeColumn, final char separator,
final Collection<String> ignoreColumnNames) {
Preconditions.checkTrue(separator == (byte) separator,
"Only separators that fulfill separator == (byte)separator are supported. "
+ "This restriction is because the parsing algorithm skips the overhead of "
+ "translating bytes to characters.");
return new CsvReaderSettings((byte) separator, ignoreColumnNames);
return new CsvReaderSettings(timeColumn, (byte) separator, ignoreColumnNames);
}
public String getTimeColumn() {
return timeColumn;
}
public void setTimeColumn(final String timeColumn) {
this.timeColumn = timeColumn;
}
public byte getSeparator() {

View File

@@ -42,7 +42,7 @@ class CsvToEntryTransformer {
int[] columns = null;
final byte[] buffer = new byte[4096 * 16];
final int keyTimestamp = Tags.STRING_COMPRESSOR.put("@timestamp");
final int keyTimestamp = Tags.STRING_COMPRESSOR.put(settings.getTimeColumn());
final int keyDuration = Tags.STRING_COMPRESSOR.put("duration");
final FastISODateParser dateParser = new FastISODateParser();

View File

@@ -66,7 +66,7 @@ public final class IngestionHandler implements Callable<Void> {
} else {
in.reset();
final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue,
CsvReaderSettings.create(','));
CsvReaderSettings.create("@timestamp", ','));
csvTransformer.readCSV(in);
}
}