fix CSV parser corrupts duration if duration is last element in line

This commit is contained in:
2019-11-14 18:40:14 +01:00
parent d9b2327f35
commit 10a7710940
3 changed files with 160 additions and 116 deletions

View File

@@ -240,7 +240,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
} else if (key == keyTimestamp) { } else if (key == keyTimestamp) {
epochMilli = dateParser.parseAsEpochMilli(line, lastSeparatorPosition + 1); epochMilli = dateParser.parseAsEpochMilli(line, lastSeparatorPosition + 1);
} else if (key == keyDuration) { } else if (key == keyDuration) {
duration = parseLong(line, lastSeparatorPosition + 1); duration = parseLong(line, lastSeparatorPosition + 1, separatorPosition);
} else if (lastSeparatorPosition + 1 < separatorPosition) { // value is not empty } else if (lastSeparatorPosition + 1 < separatorPosition) { // value is not empty
final int value = Tags.STRING_COMPRESSOR.put(line, lastSeparatorPosition + 1, final int value = Tags.STRING_COMPRESSOR.put(line, lastSeparatorPosition + 1,
separatorPosition); separatorPosition);
@@ -258,7 +258,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
return null; return null;
} }
private static long parseLong(final byte[] bytes, final int start) { private static long parseLong(final byte[] bytes, final int start, int endExclusive) {
long result = 0; long result = 0;
int i = start; int i = start;
int c = bytes[i]; int c = bytes[i];
@@ -267,7 +267,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
sign = -1; sign = -1;
i++; i++;
} }
while ((c = bytes[i]) >= 48 && c <= 57) { while (i < endExclusive && (c = bytes[i]) >= 48 && c <= 57) {
result = result * 10 + (c - 48); result = result * 10 + (c - 48);
i++; i++;
} }

View File

@@ -56,6 +56,13 @@ public class PdbTestUtil {
final Set<String> keys = entries.stream().map(Map::keySet).flatMap(Set::stream).collect(Collectors.toSet()); final Set<String> keys = entries.stream().map(Map::keySet).flatMap(Set::stream).collect(Collectors.toSet());
sendAsCsv(keys, entries);
}
public static final void sendAsCsv(Collection<String> keys, final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
final StringBuilder csv = new StringBuilder(); final StringBuilder csv = new StringBuilder();
csv.append(String.join(",", keys)); csv.append(String.join(",", keys));

View File

@@ -281,4 +281,41 @@ public class TcpIngestorTest {
"the ignored field is not returned"); "the ignored field is not returned");
} }
} }
public void testCsvIngestorHandlesDurationAtEnd() throws Exception {
String host = "someHost";
long value1 = 222;
long value2= 1;
try (TcpIngestor ingestor = new TcpIngestor(dataDirectory)) {
ingestor.start();
final Map<String, Object> entry1 = new HashMap<>();
entry1.put("@timestamp",
Instant.ofEpochMilli(1).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
entry1.put("host", host);
entry1.put("duration", value1);
final Map<String, Object> entry2 = new HashMap<>();
entry2.put("@timestamp",
Instant.ofEpochMilli(2).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
entry2.put("host", host);
entry2.put("duration", value2);
PdbTestUtil.sendAsCsv(List.of("@timestamp","host","duration"), List.of(entry1, entry2));
} catch (final Exception e) {
LOGGER.error("", e);
throw e;
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get(new Query("host=" + host, DateTimeRange.max())).singleGroup().flatMap();
Assert.assertEquals(result.size(), 4);
Assert.assertEquals(result.get(1), value1);
Assert.assertEquals(result.get(3), value2);
}
}
} }