handle corrupt json
Entries must be separated by a newline. This allows us to handle corrupt json entries, because we know that entries only start at a line beginning.
This commit is contained in:
ahr
@@ -1,80 +1,101 @@
|
||||
package org.lucares.performance.db.ingestor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SocketChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.lucares.pdbui.TcpIngestor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class PdbTestUtil {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PdbTestUtil.class);
|
||||
|
||||
private static final Map<String, Object> POISON = new HashMap<>();
|
||||
|
||||
@SafeVarargs
|
||||
public static final void send(final Map<String, Object>... entries) throws IOException, InterruptedException {
|
||||
|
||||
final LinkedBlockingDeque<Map<String, Object>> queue = new LinkedBlockingDeque<>(Arrays.asList(entries));
|
||||
queue.put(POISON);
|
||||
send(queue);
|
||||
}
|
||||
|
||||
public static final void send(final BlockingQueue<Map<String, Object>> aEntriesSupplier) throws IOException {
|
||||
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final SocketChannel channel = connect();
|
||||
|
||||
Map<String, Object> entry;
|
||||
while ((entry = aEntriesSupplier.poll()) != POISON) {
|
||||
|
||||
final StringBuilder streamData = new StringBuilder();
|
||||
streamData.append(mapper.writeValueAsString(entry));
|
||||
streamData.append("\n");
|
||||
|
||||
final ByteBuffer src = ByteBuffer.wrap(streamData.toString().getBytes(StandardCharsets.UTF_8));
|
||||
channel.write(src);
|
||||
}
|
||||
|
||||
try {
|
||||
// ugly workaround: the channel was closed too early and not all
|
||||
// data was received
|
||||
TimeUnit.MILLISECONDS.sleep(10);
|
||||
} catch (final InterruptedException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
channel.close();
|
||||
LOGGER.trace("closed sender connection");
|
||||
}
|
||||
|
||||
private static SocketChannel connect() throws IOException {
|
||||
|
||||
SocketChannel result = null;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
result = SocketChannel.open();
|
||||
result.configureBlocking(true);
|
||||
result.connect(new InetSocketAddress("127.0.0.1", TcpIngestor.PORT));
|
||||
break;
|
||||
} catch (final ConnectException e) {
|
||||
// server socket not yet ready, it should be ready any time soon
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
package org.lucares.performance.db.ingestor;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.ConnectException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.SocketChannel;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingDeque;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.lucares.pdbui.TcpIngestor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class PdbTestUtil {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PdbTestUtil.class);
|
||||
|
||||
private static final Map<String, Object> POISON = new HashMap<>();
|
||||
|
||||
@SafeVarargs
|
||||
public static final void send(final Map<String, Object>... entries) throws IOException, InterruptedException {
|
||||
|
||||
final LinkedBlockingDeque<Map<String, Object>> queue = new LinkedBlockingDeque<>(Arrays.asList(entries));
|
||||
queue.put(POISON);
|
||||
send(queue);
|
||||
}
|
||||
|
||||
public static final void send(final BlockingQueue<Map<String, Object>> aEntriesSupplier) throws IOException {
|
||||
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final SocketChannel channel = connect();
|
||||
|
||||
Map<String, Object> entry;
|
||||
while ((entry = aEntriesSupplier.poll()) != POISON) {
|
||||
|
||||
final StringBuilder streamData = new StringBuilder();
|
||||
streamData.append(mapper.writeValueAsString(entry));
|
||||
streamData.append("\n");
|
||||
|
||||
final ByteBuffer src = ByteBuffer.wrap(streamData.toString().getBytes(StandardCharsets.UTF_8));
|
||||
channel.write(src);
|
||||
}
|
||||
|
||||
try {
|
||||
// ugly workaround: the channel was closed too early and not all
|
||||
// data was received
|
||||
TimeUnit.MILLISECONDS.sleep(10);
|
||||
} catch (final InterruptedException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
channel.close();
|
||||
LOGGER.trace("closed sender connection");
|
||||
}
|
||||
|
||||
public static final void send(final String data) throws IOException {
|
||||
|
||||
final SocketChannel channel = connect();
|
||||
|
||||
final StringBuilder streamData = new StringBuilder();
|
||||
streamData.append(data);
|
||||
|
||||
final ByteBuffer src = ByteBuffer.wrap(streamData.toString().getBytes(StandardCharsets.UTF_8));
|
||||
channel.write(src);
|
||||
|
||||
try {
|
||||
// ugly workaround: the channel was closed too early and not all
|
||||
// data was received
|
||||
TimeUnit.MILLISECONDS.sleep(10);
|
||||
} catch (final InterruptedException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
channel.close();
|
||||
LOGGER.trace("closed sender connection");
|
||||
}
|
||||
|
||||
private static SocketChannel connect() throws IOException {
|
||||
|
||||
SocketChannel result = null;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
result = SocketChannel.open();
|
||||
result.configureBlocking(true);
|
||||
result.connect(new InetSocketAddress("127.0.0.1", TcpIngestor.PORT));
|
||||
break;
|
||||
} catch (final ConnectException e) {
|
||||
// server socket not yet ready, it should be ready any time soon
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import org.testng.annotations.AfterMethod;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
@Test
|
||||
public class TcpIngestorTest {
|
||||
|
||||
@@ -89,20 +91,28 @@ public class TcpIngestorTest {
|
||||
try (TcpIngestor tcpIngestor = new TcpIngestor(dataDirectory)) {
|
||||
tcpIngestor.start();
|
||||
|
||||
// this entry will be skipped, because the date is invalid
|
||||
// skipped, because the date is invalid
|
||||
final Map<String, Object> entryA = new HashMap<>();
|
||||
entryA.put("duration", 1);
|
||||
entryA.put("@timestamp", invalidDate.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
|
||||
entryA.put("host", host);
|
||||
entryA.put("tags", Collections.emptyList());
|
||||
|
||||
// skipped, because it is not valid json
|
||||
String corrupEntry = "{\"corrupt...";
|
||||
|
||||
// valid entry
|
||||
final Map<String, Object> entryB = new HashMap<>();
|
||||
entryB.put("duration", 2);
|
||||
entryB.put("@timestamp", dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
|
||||
entryB.put("host", host);
|
||||
entryB.put("tags", Collections.emptyList());
|
||||
|
||||
PdbTestUtil.send(entryA, entryB);
|
||||
final ObjectMapper objectMapper = new ObjectMapper();
|
||||
final String data = objectMapper.writeValueAsString(entryA)+"\n"+corrupEntry+"\n"+objectMapper.writeValueAsString(entryB)+"\n";
|
||||
|
||||
|
||||
PdbTestUtil.send(data);
|
||||
}
|
||||
|
||||
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
|
||||
|
||||
Reference in New Issue
Block a user