fix CSV parser corrupts duration if duration is last element in line

This commit is contained in:
2019-11-14 18:40:14 +01:00
parent d9b2327f35
commit 10a7710940
3 changed files with 160 additions and 116 deletions

View File

@@ -240,7 +240,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
} else if (key == keyTimestamp) {
epochMilli = dateParser.parseAsEpochMilli(line, lastSeparatorPosition + 1);
} else if (key == keyDuration) {
duration = parseLong(line, lastSeparatorPosition + 1);
duration = parseLong(line, lastSeparatorPosition + 1, separatorPosition);
} else if (lastSeparatorPosition + 1 < separatorPosition) { // value is not empty
final int value = Tags.STRING_COMPRESSOR.put(line, lastSeparatorPosition + 1,
separatorPosition);
@@ -258,7 +258,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
return null;
}
private static long parseLong(final byte[] bytes, final int start) {
private static long parseLong(final byte[] bytes, final int start, int endExclusive) {
long result = 0;
int i = start;
int c = bytes[i];
@@ -267,7 +267,7 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
sign = -1;
i++;
}
while ((c = bytes[i]) >= 48 && c <= 57) {
while (i < endExclusive && (c = bytes[i]) >= 48 && c <= 57) {
result = result * 10 + (c - 48);
i++;
}

View File

@@ -56,6 +56,13 @@ public class PdbTestUtil {
final Set<String> keys = entries.stream().map(Map::keySet).flatMap(Set::stream).collect(Collectors.toSet());
sendAsCsv(keys, entries);
}
public static final void sendAsCsv(Collection<String> keys, final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
final StringBuilder csv = new StringBuilder();
csv.append(String.join(",", keys));

View File

@@ -281,4 +281,41 @@ public class TcpIngestorTest {
"the ignored field is not returned");
}
}
public void testCsvIngestorHandlesDurationAtEnd() throws Exception {
String host = "someHost";
long value1 = 222;
long value2= 1;
try (TcpIngestor ingestor = new TcpIngestor(dataDirectory)) {
ingestor.start();
final Map<String, Object> entry1 = new HashMap<>();
entry1.put("@timestamp",
Instant.ofEpochMilli(1).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
entry1.put("host", host);
entry1.put("duration", value1);
final Map<String, Object> entry2 = new HashMap<>();
entry2.put("@timestamp",
Instant.ofEpochMilli(2).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
entry2.put("host", host);
entry2.put("duration", value2);
PdbTestUtil.sendAsCsv(List.of("@timestamp","host","duration"), List.of(entry1, entry2));
} catch (final Exception e) {
LOGGER.error("", e);
throw e;
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get(new Query("host=" + host, DateTimeRange.max())).singleGroup().flatMap();
Assert.assertEquals(result.size(), 4);
Assert.assertEquals(result.get(1), value1);
Assert.assertEquals(result.get(3), value2);
}
}
}