andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

send CSV file via REST

Browse Source
This commit is contained in:
Andreas Huber
2019-12-08 18:39:34 +01:00
parent f1ef13c1de
commit 85679ca0c8
12 changed files with 334 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
pdb-ui/src/test/java/org/lucares/pdbui/CsvResource.java Normal file
View File

@@ -0,0 +1,20 @@
package org.lucares.pdbui;
import java.nio.charset.StandardCharsets;
import org.springframework.core.io.ByteArrayResource;
public class CsvResource extends ByteArrayResource {
private final String filename;
public CsvResource(final String csv, final String filename) {
super(csv.getBytes(StandardCharsets.UTF_8));
this.filename = filename;
}
@Override
public String getFilename() {
return filename;
}
}

9
pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java
View File

@@ -15,7 +15,6 @@ import org.lucares.collections.LongList;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.Entries;
import org.lucares.pdb.api.Query;
import org.lucares.performance.db.BlockingQueueIterator;
import org.lucares.performance.db.PerformanceDb;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
@@ -48,13 +47,11 @@ public class CsvToEntryTransformerTest {
+ dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue\n"//
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
final ArrayBlockingQueue<Entries> queue = new ArrayBlockingQueue<Entries>(10);
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final CsvReaderSettings settings = new CsvReaderSettings(',');
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
queue.put(Entries.POISON);
db.putEntries(new BlockingQueueIterator<Entries>(queue, Entries.POISON));
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
@@ -87,13 +84,11 @@ public class CsvToEntryTransformerTest {
+ "2000-01-01T00:00:00.000Z,1,ignoreValue,ignoreValue,tagValue\n"//
+ "2000-01-01T00:00:00.001Z,2,ignoreValue,ignoreValue,tagValue\n";
final ArrayBlockingQueue<Entries> queue = new ArrayBlockingQueue<Entries>(10);
final ArrayBlockingQueue<Entries> queue = db.getQueue();
final CsvReaderSettings settings = new CsvReaderSettings(',', "ignoredColumn");
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
queue.put(Entries.POISON);
db.putEntries(new BlockingQueueIterator<Entries>(queue, Entries.POISON));
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {

29
pdb-ui/src/test/java/org/lucares/pdbui/MySpringTestConfiguration.java
View File

@@ -1,11 +1,32 @@
package org.lucares.pdbui;
import org.lucares.pdbui.MySpringConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Import;
import java.nio.file.Path;
import java.util.Properties;
@SpringBootTest
import org.lucares.utils.file.FileUtils;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.env.PropertiesPropertySource;
@Configuration
@Import(MySpringConfiguration.class)
public class MySpringTestConfiguration {
}
class TestOverrides implements ApplicationContextInitializer<ConfigurableApplicationContext> {
@Override
public void initialize(final ConfigurableApplicationContext applicationContext) {
final Properties props = new Properties();
final Path tmpdir = Path.of(System.getProperty("java.io.tmpdir")).resolve("pdb-test");
FileUtils.delete(tmpdir);
props.put("base.dir", tmpdir.toFile().getAbsolutePath());
final PropertiesPropertySource testOverrides = new PropertiesPropertySource("testOverrides", props);
applicationContext.getEnvironment().getPropertySources().addFirst(testOverrides);
}
}

85
pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java Normal file
View File

@@ -0,0 +1,85 @@
package org.lucares.pdbui;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Query;
import org.lucares.performance.db.PerformanceDb;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
import org.springframework.boot.test.web.client.TestRestTemplate;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
@RunWith(SpringRunner.class)
@ContextConfiguration(initializers = TestOverrides.class)
@SpringBootTest(classes = MySpringTestConfiguration.class, webEnvironment = WebEnvironment.RANDOM_PORT)
public class PdbControllerTest {
@Autowired
private PerformanceDb performanceDb;
@Autowired
private TestRestTemplate rest;
@Test
public void testUploadCsv() {
final OffsetDateTime dateA = OffsetDateTime.now();
final OffsetDateTime dateB = OffsetDateTime.now();
final String csv = "@timestamp,duration,tag\n"//
+ dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue\n"//
+ dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
uploadCsv(csv);
final GroupResult groupResult = performanceDb.get(new Query("tag=tagValue", DateTimeRange.ofDay(dateA)))
.singleGroup();
final LongList result = groupResult.flatMap();
System.out.println(PdbTestUtil.timeValueLongListToString(result));
Assert.assertEquals(result.size(), 4);
Assert.assertEquals(result.get(0), dateA.toInstant().toEpochMilli());
Assert.assertEquals(result.get(1), 1);
Assert.assertEquals(result.get(2), dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli());
Assert.assertEquals(result.get(3), 2);
}
private void uploadCsv(final String... csvs) {
// final TestRestTemplate rest = new TestRestTemplate();
final LinkedMultiValueMap<String, Object> parameters = new LinkedMultiValueMap<String, Object>();
int count = 0;
for (final String csv : csvs) {
parameters.add("file", new CsvResource(csv, count++ + ".csv"));
}
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
final HttpEntity<MultiValueMap<String, Object>> entity = new HttpEntity<MultiValueMap<String, Object>>(
parameters, headers);
final ResponseEntity<String> response = rest.exchange("/data", HttpMethod.POST, entity, String.class);
Assert.assertEquals("response status", HttpStatus.CREATED, response.getStatusCode());
}
}

28
pdb-ui/src/test/java/org/lucares/pdbui/PdbTestUtil.java
View File

@@ -9,6 +9,10 @@ import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -21,7 +25,7 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.lucares.pdbui.TcpIngestor;
import org.lucares.collections.LongList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +63,7 @@ public class PdbTestUtil {
sendAsCsv(keys, entries);
}
public static final void sendAsCsv(Collection<String> keys, final Collection<Map<String, Object>> entries)
public static final void sendAsCsv(final Collection<String> keys, final Collection<Map<String, Object>> entries)
throws IOException, InterruptedException {
final StringBuilder csv = new StringBuilder();
@@ -177,4 +181,24 @@ public class PdbTestUtil {
return result;
}
public static String timeValueLongListToString(final LongList timeValueLongList) {
final StringBuilder result = new StringBuilder();
int i = 0;
while (i < timeValueLongList.size()) {
final OffsetDateTime time = OffsetDateTime.ofInstant(Instant.ofEpochMilli(timeValueLongList.get(i)),
ZoneOffset.UTC);
i++;
final long value = timeValueLongList.get(i);
i++;
result.append(time.format(DateTimeFormatter.ISO_DATE_TIME));
result.append("=");
result.append(value);
result.append("\n");
}
return result.toString();
}
}