andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

TcpIngestor can handle csv files

Browse Source
This commit is contained in:
Andreas Huber
2018-10-11 18:56:16 +02:00
parent 6d4e3da672
commit 979e001efd
8 changed files with 338 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
pdb-ui/src/test/java/org/lucares/performance/db/ingestor/LongPair.java Normal file
View File

@@ -0,0 +1,72 @@
package org.lucares.performance.db.ingestor;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.lucares.collections.LongList;
final class LongPair implements Comparable<LongPair> {
private final long a, b;
public LongPair(final long a, final long b) {
super();
this.a = a;
this.b = b;
}
public static List<LongPair> fromLongList(final LongList longList) {
final List<LongPair> result = new ArrayList<>();
for (int i = 0; i < longList.size(); i += 2) {
result.add(new LongPair(longList.get(i), longList.get(i + 1)));
}
Collections.sort(result);
return result;
}
public long getA() {
return a;
}
public long getB() {
return b;
}
@Override
public String toString() {
return a + "," + b;
}
@Override
public int compareTo(final LongPair o) {
return Comparator.comparing(LongPair::getA).thenComparing(LongPair::getB).compare(this, o);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (a ^ (a >>> 32));
result = prime * result + (int) (b ^ (b >>> 32));
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final LongPair other = (LongPair) obj;
if (a != other.a)
return false;
if (b != other.b)
return false;
return true;
}
}

66
pdb-ui/src/test/java/org/lucares/performance/db/ingestor/PdbTestUtil.java
View File

@@ -6,12 +6,17 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.lucares.pdbui.TcpIngestor;
import org.slf4j.Logger;
@@ -24,15 +29,62 @@ public class PdbTestUtil {
static final Map<String, Object> POISON = new HashMap<>();
@SafeVarargs
public static final void send(final Map<String, Object>... entries) throws IOException, InterruptedException {
final LinkedBlockingDeque<Map<String, Object>> queue = new LinkedBlockingDeque<>(Arrays.asList(entries));
queue.put(POISON);
send(queue);
public static final void send(final String format, final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
switch (format) {
case "csv":
sendAsCsv(entries);
break;
case "json":
sendAsJson(entries);
break;
default:
throw new IllegalStateException("unhandled format: " + format);
}
}
public static final void send(final BlockingQueue<Map<String, Object>> aEntriesSupplier) throws IOException {
@SafeVarargs
public static final void sendAsCsv(final Map<String, Object>... entries) throws IOException, InterruptedException {
sendAsCsv(Arrays.asList(entries));
}
public static final void sendAsCsv(final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
final Set<String> keys = entries.stream().map(Map::keySet).flatMap(Set::stream).collect(Collectors.toSet());
final StringBuilder csv = new StringBuilder();
csv.append(String.join(",", keys));
csv.append("\n");
for (final Map<String, Object> entry : entries) {
final List<String> line = new ArrayList<>();
for (final String key : keys) {
final String value = String.valueOf(entry.getOrDefault(key, ""));
line.add(value);
}
csv.append(String.join(",", line));
csv.append("\n");
}
System.out.println("sending: " + csv);
send(csv.toString());
}
@SafeVarargs
public static final void sendAsJson(final Map<String, Object>... entries) throws IOException, InterruptedException {
sendAsJson(Arrays.asList(entries));
}
public static final void sendAsJson(final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
final LinkedBlockingDeque<Map<String, Object>> queue = new LinkedBlockingDeque<>(entries);
queue.put(POISON);
sendAsJson(queue);
}
public static final void sendAsJson(final BlockingQueue<Map<String, Object>> aEntriesSupplier) throws IOException {
final ObjectMapper mapper = new ObjectMapper();
final SocketChannel channel = connect();

83
pdb-ui/src/test/java/org/lucares/performance/db/ingestor/TcpIngestorTest.java
View File

@@ -11,7 +11,6 @@ import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -27,6 +26,7 @@ import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -36,70 +36,6 @@ public class TcpIngestorTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TcpIngestorTest.class);
private static final class LongPair implements Comparable<LongPair> {
private final long a, b;
public LongPair(final long a, final long b) {
super();
this.a = a;
this.b = b;
}
public static List<LongPair> fromLongList(final LongList longList) {
final List<LongPair> result = new ArrayList<>();
for (int i = 0; i < longList.size(); i += 2) {
result.add(new LongPair(longList.get(i), longList.get(i + 1)));
}
Collections.sort(result);
return result;
}
public long getA() {
return a;
}
public long getB() {
return b;
}
@Override
public String toString() {
return a + "," + b;
}
@Override
public int compareTo(final LongPair o) {
return Comparator.comparing(LongPair::getA).thenComparing(LongPair::getB).compare(this, o);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (a ^ (a >>> 32));
result = prime * result + (int) (b ^ (b >>> 32));
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final LongPair other = (LongPair) obj;
if (a != other.a)
return false;
if (b != other.b)
return false;
return true;
}
}
private Path dataDirectory;
@BeforeMethod
@@ -134,7 +70,7 @@ public class TcpIngestorTest {
entryB.put("host", host);
entryB.put("tags", Collections.emptyList());
PdbTestUtil.send(entryA, entryB);
PdbTestUtil.sendAsJson(entryA, entryB);
} catch (final Exception e) {
LOGGER.error("", e);
throw e;
@@ -201,7 +137,18 @@ public class TcpIngestorTest {
}
}
public void testRandomOrder() throws Exception {
@DataProvider
public Object[][] providerSendingFormats() {
final List<Object[]> data = new ArrayList<>();
data.add(new Object[] { "csv" });
data.add(new Object[] { "json" });
return data.toArray(Object[][]::new);
}
@Test(dataProvider = "providerSendingFormats")
public void testRandomOrder(final String format) throws Exception {
final ThreadLocalRandom rnd = ThreadLocalRandom.current();
final String host = "someHost";
@@ -232,7 +179,7 @@ public class TcpIngestorTest {
}
queue.put(PdbTestUtil.POISON);
PdbTestUtil.send(queue);
PdbTestUtil.send(format, queue);
} catch (final Exception e) {
LOGGER.error("", e);
throw e;