add file drop handler

You can define a folder and ingest files dropped into it.

pdb-ui/src/test/java/org/lucares/pdbui/FileDropHandlerTest.java

@@ -0,0 +1,116 @@
+package org.lucares.pdbui;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.util.EnumSet;
+import java.util.List;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.lucares.collections.LongList;
+import org.lucares.pdb.api.DateTimeRange;
+import org.lucares.pdb.api.Query;
+import org.lucares.pdb.api.Result;
+import org.lucares.pdbui.CsvReaderSettings.ColumnDefinitions;
+import org.lucares.pdbui.CsvReaderSettings.PostProcessors;
+import org.lucares.pdbui.domain.FileDropConfig;
+import org.lucares.pdbui.domain.FileDropSettings;
+import org.lucares.pdbui.domain.TagMatcher;
+import org.lucares.performance.db.PerformanceDb;
+import org.lucares.utils.Retry;
+import org.lucares.utils.file.FileUtils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class FileDropHandlerTest {
+
+    private Path dataDirectory;
+
+    @BeforeEach
+    public void beforeMethod() throws IOException {
+        dataDirectory = Files.createTempDirectory("pdb");
+    }
+
+    @AfterEach
+    public void afterMethod() throws IOException {
+        FileUtils.delete(dataDirectory);
+    }
+
+    @Test
+    public void testDropZipInFolder() throws Exception {
+        try (final PerformanceDb db = new PerformanceDb(dataDirectory)) {
+
+            final ColumnDefinitions columns = new ColumnDefinitions();
+            columns.ignoreColumn("user");
+            columns.ignoreColumn("trace_id");
+            columns.postProcess("project", EnumSet.of(PostProcessors.LOWER_CASE));
+            final CsvReaderSettings csvSettings = CsvReaderSettings.create("time", "duration", ";", columns);
+            csvSettings.addFirstLineMatcher(new TagMatcher("BUILD=([^ ]+)", "build"));
+            csvSettings.addFirstLineMatcher(new TagMatcher("BRANCH=([^ ]+)", "branch"));
+            final FileDropSettings setting = new FileDropSettings("{source}/{pod}/{host}/performance.*.csv",
+                    csvSettings);
+            try (final FileDropHandler handler = createFileDropHandler(db, setting)) {
+
+                final Path fileDropBaseDir = handler.getBaseDir();
+                final Path droppedFile = fileDropBaseDir.resolve("logs.zip");
+
+                /**
+                 * The zip contains one file in path /web/testpod/examplehost/ with content
+                 * 
+                 * <pre>
+                 * #BUILD=1.2.3 BRANCH=master 
+                 * time;duration;method;project;user;trace_id
+                 * 2020-01-01 00:00:00,000;42;Controller.endpoint;customerProject;alice;kaw9dyzi.1
+                 * 2020-01-01 00:00:00,001;43;Processor.endpoint;customerProject;alice;kaw9dyzi.2
+                 * 2020-01-01 00:00:00,001;44;Controller.endpoint;customerProject;alice;kaw9dyzi.2
+                 * </pre>
+                 */
+                copyResourceToFile("logs_FileDropHandlerTest_1.zip", droppedFile);
+
+                Retry.maxRetries(10).retry(() -> {
+                    final DateTimeRange range = DateTimeRange
+                            .ofDay(OffsetDateTime.of(2020, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC));
+                    final String query = "source=web and method=Controller.endpoint and branch=master and build=1.2.3";
+                    final Result result = db.get(Query.createQuery(query, range));
+                    final LongList values = result.singleGroup().flatMap();
+                    Assertions.assertEquals(values.get(1), 42, "value 1 of " + values);
+                    Assertions.assertEquals(values.get(3), 44, "value 3 of " + values);
+                    Assertions.assertEquals(values.size(), 4);
+                });
+            }
+        }
+    }
+
+    private FileDropHandler createFileDropHandler(final PerformanceDb db, final FileDropSettings... dropSettings)
+            throws Exception {
+        final Path fileDropConfigLocation = dataDirectory.resolve("drop.json");
+
+        final FileDropConfig config = new FileDropConfig();
+        config.addSettings(dropSettings);
+
+        final ObjectMapper objectMapper = new ObjectMapper();
+        objectMapper.writeValue(fileDropConfigLocation.toFile(), config);
+
+        final FileDropConfigProvider fileDropConfigProvider = new FileDropConfigProvider(
+                fileDropConfigLocation.toString());
+        final String fileDropBaseDir = dataDirectory.resolve("drop").toAbsolutePath().toString();
+        final List<FileDropFileTypeHandler> handlers = List.of(new FileDropZipHandler(db, fileDropConfigProvider));
+        return new FileDropHandler(fileDropBaseDir, handlers);
+    }
+
+    private void copyResourceToFile(final String string, final Path droppedFile) throws IOException {
+
+        try (final InputStream resourceAsStream = this.getClass().getClassLoader()
+                .getResourceAsStream("logs_FileDropHandlerTest_1.zip");
+                final FileOutputStream out = new FileOutputStream(droppedFile.toFile());) {
+            resourceAsStream.transferTo(out);
+        }
+    }
+}