TcpIngestor can handle csv files

pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java

@@ -0,0 +1,70 @@
+package org.lucares.pdbui;
+
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+import org.lucares.pdb.api.Entry;
+import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.api.TagsBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CsvToEntryTransformer implements LineToEntryTransformer {
+	private static final Logger LOGGER = LoggerFactory.getLogger(CsvToEntryTransformer.class);
+
+	private final String[] headers;
+	private final Pattern splitPattern = Pattern.compile(",");
+
+	public CsvToEntryTransformer(final String[] headers) {
+		this.headers = headers;
+	}
+
+	@Override
+	public Optional<Entry> toEntry(final String line) throws IOException {
+		Optional<Entry> result;
+		try {
+
+			final String[] columns = splitPattern.split(line);
+			if (columns.length == headers.length) {
+
+				result = createEntry(columns);
+
+			} else {
+				result = Optional.empty();
+			}
+		} catch (final Exception e) {
+			LOGGER.error("Failed to create entry from line: {}", line, e);
+			result = Optional.empty();
+		}
+		return result;
+	}
+
+	private Optional<Entry> createEntry(final String[] columns) {
+
+		OffsetDateTime date = null;
+		long duration = Long.MIN_VALUE;
+		final TagsBuilder tagsBuilder = TagsBuilder.create();
+		for (int i = 0; i < columns.length; i++) {
+
+			switch (headers[i]) {
+			case "@timestamp":
+				date = OffsetDateTime.parse(columns[i], DateTimeFormatter.ISO_ZONED_DATE_TIME);
+				break;
+			case "duration":
+				duration = Long.parseLong(columns[i]);
+				break;
+			default:
+				tagsBuilder.add(headers[i], columns[i]);
+				break;
+			}
+
+		}
+		final Tags tags = tagsBuilder.build();
+
+		final Entry entry = new Entry(date, duration, tags);
+		return Optional.of(entry);
+	}
+}

pdb-ui/src/main/java/org/lucares/pdbui/JsonToEntryTransformer.java

@@ -0,0 +1,93 @@
+package org.lucares.pdbui;
+
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import java.util.Optional;
+
+import org.lucares.pdb.api.Entry;
+import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.api.TagsBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+
+public class JsonToEntryTransformer implements LineToEntryTransformer {
+	private static final Logger LOGGER = LoggerFactory.getLogger(JsonToEntryTransformer.class);
+
+	private final TypeReference<Map<String, Object>> typeReferenceForMap = new TypeReference<>() {
+	};
+
+	private final ObjectMapper objectMapper = new ObjectMapper();
+	private final ObjectReader objectReader = objectMapper.readerFor(typeReferenceForMap);
+
+	@Override
+	public Optional<Entry> toEntry(final String line) throws IOException {
+
+		final Map<String, Object> object = objectReader.readValue(line);
+
+		final Optional<Entry> entry = createEntry(object);
+
+		return entry;
+	}
+
+	public Optional<Entry> createEntry(final Map<String, Object> map) {
+		try {
+
+			if (map.containsKey("duration") && map.containsKey("@timestamp")) {
+				final OffsetDateTime date = getDate(map);
+				final long duration = (int) map.get("duration");
+
+				final Tags tags = createTags(map);
+
+				final Entry entry = new Entry(date, duration, tags);
+				return Optional.of(entry);
+			} else {
+				LOGGER.info("Skipping invalid entry: " + map);
+				return Optional.empty();
+			}
+		} catch (final Exception e) {
+			LOGGER.error("Failed to create entry from map: " + map, e);
+			return Optional.empty();
+		}
+	}
+
+	private Tags createTags(final Map<String, Object> map) {
+		final TagsBuilder tags = TagsBuilder.create();
+		for (final java.util.Map.Entry<String, Object> e : map.entrySet()) {
+
+			final String key = e.getKey();
+			final Object value = e.getValue();
+
+			switch (key) {
+			case "@timestamp":
+			case "duration":
+				// these fields are not tags
+				break;
+			case "tags":
+				// ignore: we only support key/value tags
+				break;
+			default:
+				if (value instanceof String) {
+					tags.add(key, (String) value);
+				} else if (value != null) {
+					tags.add(key, String.valueOf(value));
+				}
+				break;
+			}
+		}
+		return tags.build();
+	}
+
+	private OffsetDateTime getDate(final Map<String, Object> map) {
+		final String timestamp = (String) map.get("@timestamp");
+
+		final OffsetDateTime date = OffsetDateTime.parse(timestamp, DateTimeFormatter.ISO_ZONED_DATE_TIME);
+		return date;
+	}
+
+}

pdb-ui/src/main/java/org/lucares/pdbui/LineToEntryTransformer.java

@@ -0,0 +1,10 @@
+package org.lucares.pdbui;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import org.lucares.pdb.api.Entry;
+
+public interface LineToEntryTransformer {
+	public Optional<Entry> toEntry(String line) throws IOException;
+}

pdb-ui/src/main/java/org/lucares/pdbui/TcpIngestor.java

@@ -19,6 +19,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
 
 import javax.annotation.PreDestroy;
 
@@ -36,9 +37,6 @@ import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
 
 import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 
 @Component
 public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
@@ -56,9 +54,6 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
 
 	public final static class Handler implements Callable<Void> {
 
-		private final TypeReference<Map<String, Object>> typeReferenceForMap = new TypeReference<Map<String, Object>>() {
-		};
-
 		final Socket clientSocket;
 		private final ArrayBlockingQueue<Entry> queue;
 
@@ -74,17 +69,29 @@ public class TcpIngestor implements Ingestor, AutoCloseable, DisposableBean {
 			LOGGER.debug("opening streams to client");
 			try (PrintWriter out = new PrintWriter(clientSocket.getOutputStream(), true);
 					BufferedReader in = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));) {
-				final ObjectMapper objectMapper = new ObjectMapper();
-				final ObjectReader objectReader = objectMapper.readerFor(typeReferenceForMap);
+				final LineToEntryTransformer transformer;
 
 				LOGGER.debug("reading from stream");
 				String line;
+
+				// determine stream type (json or csv)
+				line = in.readLine();
+				if (line.startsWith("{")) {
+					transformer = new JsonToEntryTransformer();
+					final Optional<Entry> entry = transformer.toEntry(line);
+					if (entry.isPresent()) {
+						LOGGER.debug("adding entry to queue: {}", entry);
+						queue.put(entry.get());
+					}
+				} else {
+					final String[] columnHeaders = line.split(Pattern.quote(","));
+					transformer = new CsvToEntryTransformer(columnHeaders);
+				}
+
 				while ((line = in.readLine()) != null) {
 
 					try {
-						final Map<String, Object> object = objectReader.readValue(line);
-
-						final Optional<Entry> entry = createEntry(object);
+						final Optional<Entry> entry = transformer.toEntry(line);
 
 						if (entry.isPresent()) {
 							LOGGER.debug("adding entry to queue: {}", entry);