diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java index 53eecef..1ca29af 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/Tags.java @@ -179,6 +179,10 @@ public class Tags implements Comparable { return Collections.unmodifiableList(tags); } + List getTagsUnsafe() { + return tags; + } + public void forEach(final BiConsumer keyValueConsumer) { for (final Tag tag : tags) { diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java b/pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java index 50633f4..6f0c789 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/TagsBuilder.java @@ -5,7 +5,16 @@ import java.util.List; public class TagsBuilder { - final List tags = new ArrayList<>(); + final List tags; + + public TagsBuilder() { + tags = new ArrayList<>(); + } + + public TagsBuilder(final Tags tags) { + this.tags = new ArrayList<>(); + this.tags.addAll(tags.getTagsUnsafe()); + } public static TagsBuilder create() { return new TagsBuilder(); @@ -31,4 +40,5 @@ public class TagsBuilder { public Tags build() { return Tags.create(tags); } + } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java b/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java index 4f4c2df..29587f9 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java @@ -1,8 +1,10 @@ package org.lucares.pdbui; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.lucares.utils.Preconditions; @@ -12,6 +14,8 @@ public class CsvReaderSettings { private Set ignoreColumnNames = new HashSet(); + private final Map additionalTags = new HashMap(); + private String timeColumn; public CsvReaderSettings() { @@ -73,4 +77,12 @@ public class CsvReaderSettings { return ignoreColumnNames.contains(columnName); } + public void putAdditionalTag(final String field, final String value) { + additionalTags.put(field, value); + } + + public Map getAdditionalTags() { + return Map.copyOf(additionalTags); + } + } diff --git a/pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java b/pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java index 125a2b4..2d2bf3a 100644 --- a/pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java +++ b/pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java @@ -13,8 +13,12 @@ import org.lucares.pdb.api.TagsBuilder; import org.lucares.pdb.datastore.Entries; import org.lucares.pdb.datastore.Entry; import org.lucares.pdbui.date.FastISODateParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class CsvToEntryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(CsvToEntryTransformer.class); + /** * Column header names starting with "-" will be ignored. */ @@ -48,6 +52,8 @@ class CsvToEntryTransformer { final int keyDuration = Tags.STRING_COMPRESSOR.put("duration"); final FastISODateParser dateParser = new FastISODateParser(); + final Tags additionalTags = initAditionalTags(); + while ((read = in.read(buffer)) >= 0) { offsetInBuffer = 0; @@ -61,7 +67,7 @@ class CsvToEntryTransformer { if (columns != null) { final Entry entry = handleCsvLine(columns, line, bytesInLine, separatorPositions, keyTimestamp, - keyDuration, dateParser); + keyDuration, dateParser, additionalTags); if (entry != null) { entries.add(entry); } @@ -91,7 +97,7 @@ class CsvToEntryTransformer { } } final Entry entry = handleCsvLine(columns, line, bytesInLine, separatorPositions, keyTimestamp, keyDuration, - dateParser); + dateParser, additionalTags); if (entry != null) { entries.add(entry); } @@ -100,6 +106,16 @@ class CsvToEntryTransformer { entries.waitUntilFlushed(5, TimeUnit.MINUTES); } + private Tags initAditionalTags() { + final TagsBuilder tags = new TagsBuilder(); + for (final java.util.Map.Entry entry : settings.getAdditionalTags().entrySet()) { + final int field = Tags.STRING_COMPRESSOR.put(entry.getKey()); + final int value = Tags.STRING_COMPRESSOR.put(entry.getValue()); + tags.add(field, value); + } + return tags.build(); + } + private int[] handleCsvHeaderLine(final byte[] line, final int bytesInLine, final IntList separatorPositions) { final int[] columns = new int[separatorPositions.size()]; @@ -125,12 +141,12 @@ class CsvToEntryTransformer { private static Entry handleCsvLine(final int[] columns, final byte[] line, final int bytesInLine, final IntList separatorPositions, final int keyTimestamp, final int keyDuration, - final FastISODateParser dateParser) { + final FastISODateParser dateParser, final Tags additionalTags) { try { if (separatorPositions.size() != columns.length) { return null; } - final TagsBuilder tagsBuilder = new TagsBuilder(); + final TagsBuilder tagsBuilder = new TagsBuilder(additionalTags); int lastSeparatorPosition = -1; final int size = separatorPositions.size(); long epochMilli = -1; @@ -155,8 +171,7 @@ class CsvToEntryTransformer { final Tags tags = tagsBuilder.build(); return new Entry(epochMilli, duration, tags); } catch (final RuntimeException e) { - TcpIngestor.LOGGER.debug( - "ignoring invalid line '" + new String(line, 0, bytesInLine, StandardCharsets.UTF_8) + "'", e); + LOGGER.debug("ignoring invalid line '" + new String(line, 0, bytesInLine, StandardCharsets.UTF_8) + "'", e); } return null; } diff --git a/pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java b/pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java index 53af5ef..27c99fb 100644 --- a/pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java +++ b/pdb-ui/src/test/java/org/lucares/pdbui/PdbControllerTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.lucares.collections.LongList; import org.lucares.pdb.api.DateTimeRange; -import org.lucares.pdb.api.GroupResult; import org.lucares.pdb.api.Query; import org.lucares.performance.db.PerformanceDb; import org.springframework.beans.factory.annotation.Autowired; @@ -48,6 +47,8 @@ public class PdbControllerTest { @Test public void testUploadCsv() throws InterruptedException { + final String additionalColumn = "additionalColumn"; + final String additionalValue = "additionalValue"; final String ignoredColumn = "ignoredColumn"; final String timeColumn = "time"; final OffsetDateTime dateA = OffsetDateTime.now(); @@ -58,23 +59,33 @@ public class PdbControllerTest { + dateB.format(DateTimeFormatter.ISO_ZONED_DATE_TIME) + ",2,tagValue,ignoredValue\n"; final CsvReaderSettings settings = CsvReaderSettings.create(timeColumn, ',', ignoredColumn); + settings.putAdditionalTag(additionalColumn, additionalValue); uploadCsv(settings, csv); { - final GroupResult groupResult = performanceDb.get(new Query("tag=tagValue", DateTimeRange.ofDay(dateA))) - .singleGroup(); - final LongList result = groupResult.flatMap(); - System.out.println(PdbTestUtil.timeValueLongListToString(result)); - Assertions.assertEquals(4, result.size()); + final LongList resultTagValue = performanceDb.get(new Query("tag=tagValue", DateTimeRange.ofDay(dateA))) + .singleGroup().flatMap(); + final LongList resultAdditionalValue = performanceDb + .get(new Query(additionalColumn + "=" + additionalValue, DateTimeRange.ofDay(dateA))).singleGroup() + .flatMap(); + System.out.println(PdbTestUtil.timeValueLongListToString(resultTagValue)); - Assertions.assertEquals(dateA.toInstant().toEpochMilli(), result.get(0)); - Assertions.assertEquals(1, result.get(1)); + Assertions.assertEquals(resultTagValue, resultAdditionalValue, + "results from queries tag=value should be equal to results from query for additionalColumn=additionalValue"); - Assertions.assertEquals(dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), result.get(2)); - Assertions.assertEquals(2, result.get(3)); + Assertions.assertEquals(4, resultTagValue.size()); + + Assertions.assertEquals(dateA.toInstant().toEpochMilli(), resultTagValue.get(0)); + Assertions.assertEquals(1, resultTagValue.get(1)); + + Assertions.assertEquals(dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli(), + resultTagValue.get(2)); + Assertions.assertEquals(2, resultTagValue.get(3)); } { final List fields = performanceDb.getFields(DateTimeRange.max()); Assertions.assertTrue(!fields.contains(ignoredColumn), "ignoredColumn not in fields. fields: " + fields); + Assertions.assertTrue(fields.contains(additionalColumn), + additionalColumn + " expected in fields. Fields were: " + fields); } }