make time column configurable
Adding the header of the time column to the CsvReaderSettings. This is necessary to parse arbitrary CSV files.
This commit is contained in:
@@ -12,30 +12,45 @@ public class CsvReaderSettings {
|
||||
|
||||
private Set<String> ignoreColumnNames = new HashSet<String>();
|
||||
|
||||
private String timeColumn;
|
||||
|
||||
public CsvReaderSettings() {
|
||||
separator = ',';
|
||||
timeColumn = "@timestamp";
|
||||
}
|
||||
|
||||
private CsvReaderSettings(final byte separator, final Collection<String> ignoreColumns) {
|
||||
private CsvReaderSettings(final String timeColumn, final byte separator, final Collection<String> ignoreColumns) {
|
||||
|
||||
this.timeColumn = timeColumn;
|
||||
this.separator = separator;
|
||||
this.ignoreColumnNames.addAll(ignoreColumns);
|
||||
}
|
||||
|
||||
public static CsvReaderSettings create(final byte separator, final String... ignoreColumnNames) {
|
||||
return new CsvReaderSettings(separator, List.of(ignoreColumnNames));
|
||||
public static CsvReaderSettings create(final String timeColumn, final byte separator,
|
||||
final String... ignoreColumnNames) {
|
||||
return new CsvReaderSettings(timeColumn, separator, List.of(ignoreColumnNames));
|
||||
}
|
||||
|
||||
public static CsvReaderSettings create(final char separator, final String... ignoreColumnNames) {
|
||||
return CsvReaderSettings.create(separator, List.of(ignoreColumnNames));
|
||||
public static CsvReaderSettings create(final String timeColumn, final char separator,
|
||||
final String... ignoreColumnNames) {
|
||||
return CsvReaderSettings.create(timeColumn, separator, List.of(ignoreColumnNames));
|
||||
}
|
||||
|
||||
public static CsvReaderSettings create(final char separator, final Collection<String> ignoreColumnNames) {
|
||||
public static CsvReaderSettings create(final String timeColumn, final char separator,
|
||||
final Collection<String> ignoreColumnNames) {
|
||||
Preconditions.checkTrue(separator == (byte) separator,
|
||||
"Only separators that fulfill separator == (byte)separator are supported. "
|
||||
+ "This restriction is because the parsing algorithm skips the overhead of "
|
||||
+ "translating bytes to characters.");
|
||||
return new CsvReaderSettings((byte) separator, ignoreColumnNames);
|
||||
return new CsvReaderSettings(timeColumn, (byte) separator, ignoreColumnNames);
|
||||
}
|
||||
|
||||
public String getTimeColumn() {
|
||||
return timeColumn;
|
||||
}
|
||||
|
||||
public void setTimeColumn(final String timeColumn) {
|
||||
this.timeColumn = timeColumn;
|
||||
}
|
||||
|
||||
public byte getSeparator() {
|
||||
|
||||
@@ -42,7 +42,7 @@ class CsvToEntryTransformer {
|
||||
|
||||
int[] columns = null;
|
||||
final byte[] buffer = new byte[4096 * 16];
|
||||
final int keyTimestamp = Tags.STRING_COMPRESSOR.put("@timestamp");
|
||||
final int keyTimestamp = Tags.STRING_COMPRESSOR.put(settings.getTimeColumn());
|
||||
final int keyDuration = Tags.STRING_COMPRESSOR.put("duration");
|
||||
final FastISODateParser dateParser = new FastISODateParser();
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ public final class IngestionHandler implements Callable<Void> {
|
||||
} else {
|
||||
in.reset();
|
||||
final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue,
|
||||
CsvReaderSettings.create(','));
|
||||
CsvReaderSettings.create("@timestamp", ','));
|
||||
csvTransformer.readCSV(in);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user