extend CsvReaderSettings by list of columns that will not be indexed

pdb-ui/src/test/java/org/lucares/pdbui/CsvToEntryTransformerTest.java

@@ -0,0 +1,105 @@
+package org.lucares.pdbui;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.lucares.collections.LongList;
+import org.lucares.pdb.api.DateTimeRange;
+import org.lucares.pdb.api.Entries;
+import org.lucares.pdb.api.Query;
+import org.lucares.performance.db.BlockingQueueIterator;
+import org.lucares.performance.db.PerformanceDb;
+import org.lucares.utils.file.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test
+public class CsvToEntryTransformerTest {
+
+    private Path dataDirectory;
+
+    @BeforeMethod
+    public void beforeMethod() throws IOException {
+        dataDirectory = Files.createTempDirectory("pdb");
+    }
+
+    @AfterMethod
+    public void afterMethod() throws IOException {
+        FileUtils.delete(dataDirectory);
+    }
+
+    public void testIngest() throws IOException, InterruptedException {
+        final OffsetDateTime dateA = OffsetDateTime.now();
+        final OffsetDateTime dateB = OffsetDateTime.now();
+
+        try (final PerformanceDb db = new PerformanceDb(dataDirectory)) {
+
+            final String csv = "@timestamp,duration,tag\n"//
+                    + dateA.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",1,tagValue\n"//
+                    + dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue\n";
+
+            final ArrayBlockingQueue<Entries> queue = new ArrayBlockingQueue<Entries>(10);
+            final CsvReaderSettings settings = new CsvReaderSettings(',');
+            final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
+            csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
+            queue.put(Entries.POISON);
+
+            db.putEntries(new BlockingQueueIterator<Entries>(queue, Entries.POISON));
+        }
+
+        try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+            final LongList result = db.get(new Query("tag=tagValue", DateTimeRange.max())).singleGroup().flatMap();
+            Assert.assertEquals(result.size(), 4);
+
+            Assert.assertEquals(result.get(0), dateA.toInstant().toEpochMilli());
+            Assert.assertEquals(result.get(1), 1);
+
+            Assert.assertEquals(result.get(2), dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli());
+            Assert.assertEquals(result.get(3), 2);
+        }
+    }
+
+    /**
+     * There are two ways to ignore columns:
+     * <ol>
+     * <li>by specifying the column header name in the {@link CsvReaderSettings}.
+     * <li>by adding the prefix &quot;-&quot; to the column header name.
+     * </ol>
+     * 
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    public void testIgnoreColumns() throws IOException, InterruptedException {
+
+        try (final PerformanceDb db = new PerformanceDb(dataDirectory)) {
+
+            final String csv = "@timestamp,duration,ignoredColumn,-otherIgnoredColumn,tag\n"//
+                    + "2000-01-01T00:00:00.000Z,1,ignoreValue,ignoreValue,tagValue\n"//
+                    + "2000-01-01T00:00:00.001Z,2,ignoreValue,ignoreValue,tagValue\n";
+
+            final ArrayBlockingQueue<Entries> queue = new ArrayBlockingQueue<Entries>(10);
+            final CsvReaderSettings settings = new CsvReaderSettings(',', "ignoredColumn");
+            final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, settings);
+            csvToEntryTransformer.readCSV(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8)));
+            queue.put(Entries.POISON);
+
+            db.putEntries(new BlockingQueueIterator<Entries>(queue, Entries.POISON));
+        }
+
+        try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+            final List<String> availableFields = db.getFields(DateTimeRange.max());
+            Assert.assertEquals(availableFields.toString(), List.of("tag").toString(),
+                    "the ignored field is not returned");
+        }
+    }
+}