From c6d7f97628c09c9d0cebc6b68f0571058eb96473 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 27 Nov 2020 19:49:22 +0100 Subject: [PATCH] serialize the ingestion of logfiles performance improvement of roughly 40% --- .../org/lucares/pdbui/CsvUploadHandler.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java b/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java index 47b4394..2d24272 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/CsvUploadHandler.java @@ -42,11 +42,18 @@ public class CsvUploadHandler implements PropertyKeys, DisposableBean { try { final ArrayBlockingQueue queue = performanceDb.getQueue(); for (final MultipartFile file : files) { - final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); - try (InputStream in = file.getInputStream()) { - csvToEntryTransformer.readCSV(in); - } catch (final Exception e) { - LOGGER.error("csv ingestion failed", e); + + // insert one file at a time. This improves performance, because the likelihood + // of values having the same tags is greatly increased. In my tests this + // improved the + // ingestion performance fom 1.1m to 1.55m values per second on average + synchronized (this) { + final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings); + try (InputStream in = file.getInputStream()) { + csvToEntryTransformer.readCSV(in); + } catch (final Exception e) { + LOGGER.error("csv ingestion failed", e); + } } } } catch (final RuntimeException e) {