diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java b/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java index 2d5e464..47b4394 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java @@ -1,18 +1,14 @@ package org.lucares.pdbui; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.Files; +import java.io.InputStream; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -22,7 +18,6 @@ import org.lucares.utils.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.web.multipart.MultipartFile; @@ -31,56 +26,32 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean { private static final Logger LOGGER = LoggerFactory.getLogger(CsvUploadHandler.class); - private final Path tmpDir; - private final ExecutorService threadPool = Executors.newFixedThreadPool(2); private final PerformanceDb performanceDb; - public CsvUploadHandler(@Value("${" + TMP_DIR + "}") final String tmpDir, final PerformanceDb performanceDb) - throws IOException { - this.tmpDir = Paths.get(tmpDir).resolve("uploads"); - Files.createDirectories(this.tmpDir); + public CsvUploadHandler(final PerformanceDb performanceDb) throws IOException { this.performanceDb = performanceDb; } - public void ingest(final List files, final CsvReaderSettings settings, - final boolean waitUntilFinished) + public void ingest(final List files, final CsvReaderSettings settings) throws IllegalStateException, IOException, InterruptedException, ExecutionException, TimeoutException { final List tmpFiles = new ArrayList(); try { - for (final MultipartFile file : files) { - final Path tmpFile = tmpDir.resolve(UUID.randomUUID().toString()); - tmpFiles.add(tmpFile); - LOGGER.debug("writing uploaded file to {}", tmpFile); - file.transferTo(tmpFile); - } - } catch (RuntimeException | IOException e) { - FileUtils.deleteSilently(tmpFiles); - throw e; - } - - final Future future = threadPool.submit(() -> { final ArrayBlockingQueue queue = performanceDb.getQueue(); - for (final Path tmpFile : tmpFiles) { - try { - final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); - try (FileInputStream in = new FileInputStream(tmpFile.toFile())) { - csvToEntryTransformer.readCSV(in); - } - - LOGGER.debug("delete uploaded file {}", tmpFile); - Files.delete(tmpFile); - + for (final MultipartFile file : files) { + final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); + try (InputStream in = file.getInputStream()) { + csvToEntryTransformer.readCSV(in); } catch (final Exception e) { LOGGER.error("csv ingestion failed", e); } } - }); - if (waitUntilFinished) { - future.get(1, TimeUnit.HOURS); + } catch (final RuntimeException e) { + FileUtils.deleteSilently(tmpFiles); + throw e; } } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java index 15add04..0aa8fde 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java @@ -298,11 +298,10 @@ public class PdbController implements HardcodedValues, PropertyKeys { @ResponseBody @ResponseStatus(code = HttpStatus.CREATED) public String handleCsvFileUpload(@RequestParam("file") final MultipartFile[] files, - @RequestPart("settings") final CsvReaderSettings csvReaderSettings, - @RequestParam(name = "waitUntilFinished", defaultValue = "false") final boolean waitUntilFinished) + @RequestPart("settings") final CsvReaderSettings csvReaderSettings) throws IllegalStateException, IOException, InterruptedException, ExecutionException, TimeoutException { - csvUploadHandler.ingest(List.of(files), csvReaderSettings, waitUntilFinished); + csvUploadHandler.ingest(List.of(files), csvReaderSettings); return ""; // return value might become a job id that can be used to cancel, or observe // status } diff --git a/plogdownloader/performance_to_json.py b/plogdownloader/performance_to_json.py index 5ef20c3..8f3450f 100644 --- a/plogdownloader/performance_to_json.py +++ b/plogdownloader/performance_to_json.py @@ -216,13 +216,12 @@ def send_csv(zip, file_in_zip, csvSettings): 'settings': ('csvReaderSettings.json', csvSettings , 'application/json') }) - r = requests.post('http://localhost:17333/api/data?waitUntilFinished=true', data=m, + r = requests.post('http://localhost:17333/api/data', data=m, headers={ 'Accept': 'text/plain, application/json, application/*+json, */*', 'Content-Type': m.content_type.replace("form-data", "mixed;charset=UTF-8") }) - print("response:") - print(r) + print("response: %s" % r) finally: shutil.rmtree(tmp_folder)