From 60f1a7981695ddce532bfd1bb1017c9c4a3a1a2a Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Sun, 8 Dec 2019 20:20:13 +0100 Subject: [PATCH] add settings to file upload This makes it possible to define properties for the uploaded CSV files. Currently we can define the separator and which columns are to be ignored. --- .../org/lucares/pdbui/CsvReaderSettings.java | 45 ++++++++++++------ .../org/lucares/pdbui/IngestionHandler.java | 3 +- .../java/org/lucares/pdbui/PdbController.java | 8 ++-- .../pdbui/CsvToEntryTransformerTest.java | 4 +- .../java/org/lucares/pdbui/JsonResource.java | 21 +++++++++ .../pdbui/MySpringTestConfiguration.java | 5 +- .../org/lucares/pdbui/PdbControllerTest.java | 46 +++++++++++-------- .../{CsvResource.java => StringResource.java} | 6 +-- 8 files changed, 96 insertions(+), 42 deletions(-) create mode 100644 pdb-ui/src/test/java/org/lucares/pdbui/JsonResource.java rename pdb-ui/src/test/java/org/lucares/pdbui/{CsvResource.java => StringResource.java} (60%) diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java b/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java index feed707..2f3ab36 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java @@ -8,37 +8,54 @@ import java.util.Set; import org.lucares.utils.Preconditions; public class CsvReaderSettings { - private final byte separator; + private byte separator; - private final Set ignoreColumnNames = new HashSet(); + private Set ignoreColumnNames = new HashSet(); - public CsvReaderSettings(final byte separator, final String... ignoreColumnNames) { - this(separator, List.of(ignoreColumnNames)); + public CsvReaderSettings() { + separator = ','; } - public CsvReaderSettings(final char separator, final String... ignoreColumnNames) { - this(separator, List.of(ignoreColumnNames)); + private CsvReaderSettings(final byte separator, final Collection ignoreColumns) { + + this.separator = separator; + this.ignoreColumnNames.addAll(ignoreColumns); } - public CsvReaderSettings(final char separator, final Collection ignoreColumnNames) { - this((byte) separator, ignoreColumnNames); + public static CsvReaderSettings create(final byte separator, final String... ignoreColumnNames) { + return new CsvReaderSettings(separator, List.of(ignoreColumnNames)); + } + + public static CsvReaderSettings create(final char separator, final String... ignoreColumnNames) { + return CsvReaderSettings.create(separator, List.of(ignoreColumnNames)); + } + + public static CsvReaderSettings create(final char separator, final Collection ignoreColumnNames) { Preconditions.checkTrue(separator == (byte) separator, "Only separators that fulfill separator == (byte)separator are supported. " + "This restriction is because the parsing algorithm skips the overhead of " + "translating bytes to characters."); - } - - public CsvReaderSettings(final byte separator, final Collection ignoreColumns) { - - this.separator = separator; - this.ignoreColumnNames.addAll(ignoreColumns); + return new CsvReaderSettings((byte) separator, ignoreColumnNames); } public byte getSeparator() { return separator; } + public void setSeparator(final byte separator) { + this.separator = separator; + } + + public Set getIgnoreColumnNames() { + return ignoreColumnNames; + } + + public void setIgnoreColumnNames(final Set ignoreColumnNames) { + this.ignoreColumnNames = ignoreColumnNames; + } + public boolean isIgnoredColumn(final String columnName) { return ignoreColumnNames.contains(columnName); } + } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java b/pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java index d03cda9..786f002 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/IngestionHandler.java @@ -65,7 +65,8 @@ public final class IngestionHandler implements Callable { handleInputStream(gzip); } else { in.reset(); - final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue, new CsvReaderSettings(',')); + final CsvToEntryTransformer csvTransformer = new CsvToEntryTransformer(queue, + CsvReaderSettings.create(',')); csvTransformer.readCSV(in); } } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 9fee162..4747358 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -54,6 +54,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.multipart.MultipartFile; @@ -322,13 +323,14 @@ public class PdbController implements HardcodedValues, PropertyKeys { return result; } - @PostMapping(path = "/data") + @PostMapping(path = "/data", consumes = MediaType.MULTIPART_MIXED_VALUE) @ResponseBody @ResponseStatus(code = HttpStatus.CREATED) - public String handleCsvFileUpload(@RequestParam("file") final MultipartFile[] files) + public String handleCsvFileUpload(@RequestParam("file") final MultipartFile[] files, + @RequestPart("settings") final CsvReaderSettings csvReaderSettings) throws IllegalStateException, IOException { - csvUploadHandler.ingest(List.of(files), new CsvReaderSettings(',')); + csvUploadHandler.ingest(List.of(files), csvReaderSettings); return ""; // return value might become a job id that can be used to cancel, or observe // status } diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java b/pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java index b5d9cae..c68ba7b 100644 --- a/pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java +++ b/pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java @@ -48,7 +48,7 @@ public class CsvToEntryTransformerTest { + dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n"; final ArrayBlockingQueue queue = db.getQueue(); - final CsvReaderSettings settings = new CsvReaderSettings(','); + final CsvReaderSettings settings = CsvReaderSettings.create(','); final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8))); queue.put(Entries.POISON); @@ -85,7 +85,7 @@ public class CsvToEntryTransformerTest { + "2000-01-01T00:00:00.001Z,2,ignoreValue,ignoreValue,tagValue\n"; final ArrayBlockingQueue queue = db.getQueue(); - final CsvReaderSettings settings = new CsvReaderSettings(',', "ignoredColumn"); + final CsvReaderSettings settings = CsvReaderSettings.create(',', "ignoredColumn"); final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8))); queue.put(Entries.POISON); diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/JsonResource.java b/pdb-ui/src/test/java/org/lucares/pdbui/JsonResource.java new file mode 100644 index 0000000..02121c8 --- /dev/null +++ b/pdb-ui/src/test/java/org/lucares/pdbui/JsonResource.java @@ -0,0 +1,21 @@ +package org.lucares.pdbui; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class JsonResource extends StringResource { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public JsonResource(final Object obj, final String filename) { + super(stringify(obj), filename); + } + + private static String stringify(final Object obj) { + try { + return OBJECT_MAPPER.writeValueAsString(obj); + } catch (final JsonProcessingException e) { + throw new RuntimeException("failed to serialize object", e); + } + } + +} diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/MySpringTestConfiguration.java b/pdb-ui/src/test/java/org/lucares/pdbui/MySpringTestConfiguration.java index 80e1242..c4f4991 100644 --- a/pdb-ui/src/test/java/org/lucares/pdbui/MySpringTestConfiguration.java +++ b/pdb-ui/src/test/java/org/lucares/pdbui/MySpringTestConfiguration.java @@ -1,5 +1,6 @@ package org.lucares.pdbui; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Properties; @@ -23,7 +24,9 @@ class TestOverrides implements ApplicationContextInitializer fields = performanceDb.getFields(DateTimeRange.max()); + Assert.assertTrue("ignoredColumn not in fields. fields: " + fields, !fields.contains(ignoredColumn)); + } } - private void uploadCsv(final String... csvs) { - // final TestRestTemplate rest = new TestRestTemplate(); + private void uploadCsv(final CsvReaderSettings csvReaderSettings, final String... csvs) { + final LinkedMultiValueMap parameters = new LinkedMultiValueMap(); int count = 0; for (final String csv : csvs) { - parameters.add("file", new CsvResource(csv, count++ + ".csv")); + parameters.add("file", new StringResource(csv, count++ + ".csv")); } + parameters.add("settings", new JsonResource(csvReaderSettings, "csvReaderSettings.json")); + final HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.setContentType(MediaType.MULTIPART_MIXED); final HttpEntity> entity = new HttpEntity>( parameters, headers); diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/CsvResource.java b/pdb-ui/src/test/java/org/lucares/pdbui/StringResource.java similarity index 60% rename from pdb-ui/src/test/java/org/lucares/pdbui/CsvResource.java rename to pdb-ui/src/test/java/org/lucares/pdbui/StringResource.java index 350f42a..da934c3 100644 --- a/pdb-ui/src/test/java/org/lucares/pdbui/CsvResource.java +++ b/pdb-ui/src/test/java/org/lucares/pdbui/StringResource.java @@ -4,12 +4,12 @@ import java.nio.charset.StandardCharsets; import org.springframework.core.io.ByteArrayResource; -public class CsvResource extends ByteArrayResource { +public class StringResource extends ByteArrayResource { private final String filename; - public CsvResource(final String csv, final String filename) { - super(csv.getBytes(StandardCharsets.UTF_8)); + public StringResource(final String string, final String filename) { + super(string.getBytes(StandardCharsets.UTF_8)); this.filename = filename; }