andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

create Tags in StringCompressor instead of use StringCompressor in Tags

Browse Source
This commit is contained in:
Andreas Huber
2021-08-25 20:00:31 +02:00
parent b01567c82e
commit 5d2fdb4820
8 changed files with 171 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -23,6 +23,7 @@ import javax.swing.JTextArea;
import javax.swing.JTextField;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -36,7 +37,6 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.Proposal;
import org.junit.jupiter.api.Assertions;
import org.lucares.utils.CollectionUtils;
import org.lucares.utils.DateUtils;
import org.lucares.utils.file.FileUtils;
@@ -66,11 +66,13 @@ public class DataStoreTest {
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags eagleTim = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags pigeonJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "pigeon", "name",
"Jennifer");
final Tags flamingoJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "flamingo", "name",
"Jennifer");
final Tags labradorJenny = Tags.STRING_COMPRESSOR.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.STRING_COMPRESSOR.createAndAddToDictionary("dog", "labrador", "name", "Tim");
tagsToBlockStorageRootBlockNumber = new HashMap<>();
tagsToBlockStorageRootBlockNumber.put(eagleTim, dataStore.createNewFile(partitionId, eagleTim));
@@ -115,8 +117,10 @@ public class DataStoreTest {
dataStore = new DataStore(dataDirectory);
tagsToBlockStorageRootBlockNumber = new LinkedHashMap<>();
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags pigeonJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "pigeon", "name",
"Jennifer");
final Tags flamingoJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "flamingo", "name",
"Jennifer");
final ParititionId partitionId = new ParititionId("partitionA");
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(partitionId, pigeonJennifer));
@@ -130,7 +134,7 @@ public class DataStoreTest {
public void testBlockAlignment() throws IOException {
dataStore = new DataStore(dataDirectory);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTim = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final long eagleTimBlockOffset = dataStore.createNewFile(new ParititionId("partitionA"), eagleTim);
Assertions.assertEquals(0, eagleTimBlockOffset % BSFile.BLOCK_SIZE);
}
@@ -183,18 +187,28 @@ public class DataStoreTest {
final DateTimeRange dateRange = DateTimeRange.relativeHours(1);
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name", "Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name", "Jennifer"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three",
"name", "Tim"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two",
"name", "Jennifer"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one",
"name", "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three",
"name", "Jenny"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three",
"name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name",
"Timothy"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name",
"Paul"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three",
"name", "Jane"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name",
"Sam"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name",
"John"));
tags.forEach(t -> dataStore.createNewFile(partitionId, t));
@@ -208,7 +222,7 @@ public class DataStoreTest {
final long timestamp = DateUtils.getDate(2016, 1, 1, 13, 1, 1).toInstant().toEpochMilli();
final Tags tags = Tags.createAndAddToDictionary("myKey", "myValue");
final Tags tags = Tags.STRING_COMPRESSOR.createAndAddToDictionary("myKey", "myValue");
dataStore.write(timestamp, tags, 1);
dataStore.write(timestamp, tags, 2);
@@ -222,21 +236,28 @@ public class DataStoreTest {
try (final DataStore dataStore = new DataStore(dir)) {
final List<Tags> tags = Arrays.asList(
Tags.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two", "name",
"Jennifer"),
Tags.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one", "name",
"Jennifer"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "eagle", "age", "three",
"name", "Tim"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "pigeon", "age", "two",
"name", "Jennifer"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "bird", "subtype", "flamingo", "age", "one",
"name", "Jennifer"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name",
"Jenny"),
Tags.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age", "three", "name", "Tim"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age",
"three", "name", "Jenny"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "dog", "subtype", "labrador", "age",
"three", "name", "Tim"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one", "name", "Timothy"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two", "name", "Paul"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three", "name", "Jane"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "Sam"),
Tags.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four", "name", "John"));
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "one",
"name", "Timothy"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "tiger", "age", "two",
"name", "Paul"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "three",
"name", "Jane"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four",
"name", "Sam"),
Tags.STRING_COMPRESSOR.createAndAddToDictionary("type", "cat", "subtype", "lion", "age", "four",
"name", "John"));
final DateTimeRange dateRange = DateTimeRange.relativeMillis(0);
final ParititionId partitionId = DateIndexExtension.toPartitionIds(dateRange).get(0);

29
data-store/src/test/java/org/lucares/pdb/datastore/internal/ProposerTest.java
View File

@@ -8,6 +8,7 @@ import java.util.Collections;
import java.util.List;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.lucares.pdb.api.DateTimeRange;
@@ -15,7 +16,6 @@ import org.lucares.pdb.api.QueryWithCaretMarker;
import org.lucares.pdb.api.QueryWithCaretMarker.ResultMode;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Proposal;
import org.junit.jupiter.api.Assertions;
import org.lucares.utils.CollectionUtils;
import org.lucares.utils.file.FileUtils;
@@ -44,18 +44,23 @@ public class ProposerTest {
dateRange = DateTimeRange.now();
final ParititionId now = DateIndexExtension.toPartitionIds(dateRange).get(0);
final Tags eagleTim = Tags.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.createAndAddToDictionary("bird", "pigeon", "name", "Jennifer");
final Tags flamingoJennifer = Tags.createAndAddToDictionary("bird", "flamingo", "name", "Jennifer");
final Tags labradorJenny = Tags.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags eagleTim = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "eagle", "name", "Tim");
final Tags eagleTimothy = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "eagle", "name", "Timothy");
final Tags pigeonJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "pigeon", "name",
"Jennifer");
final Tags flamingoJennifer = Tags.STRING_COMPRESSOR.createAndAddToDictionary("bird", "flamingo", "name",
"Jennifer");
final Tags labradorJenny = Tags.STRING_COMPRESSOR.createAndAddToDictionary("dog", "labrador", "name", "Jenny");
final Tags labradorTim = Tags.STRING_COMPRESSOR.createAndAddToDictionary("dog", "labrador", "name", "Tim");
final Tags methodA = Tags.createAndAddToDictionary("method", "FooController.doImportantStuff", "source", "web");
final Tags methodB = Tags.createAndAddToDictionary("method", "FooService.doImportantStuff", "source",
"service");
final Tags methodC = Tags.createAndAddToDictionary("method", "BarController.doBoringStuff", "source", "web");
final Tags methodD = Tags.createAndAddToDictionary("method", "FooBarService.doOtherStuff", "source", "service");
final Tags methodA = Tags.STRING_COMPRESSOR.createAndAddToDictionary("method", "FooController.doImportantStuff",
"source", "web");
final Tags methodB = Tags.STRING_COMPRESSOR.createAndAddToDictionary("method", "FooService.doImportantStuff",
"source", "service");
final Tags methodC = Tags.STRING_COMPRESSOR.createAndAddToDictionary("method", "BarController.doBoringStuff",
"source", "web");
final Tags methodD = Tags.STRING_COMPRESSOR.createAndAddToDictionary("method", "FooBarService.doOtherStuff",
"source", "service");
dataStore.createNewFile(now, eagleTim);
dataStore.createNewFile(now, eagleTimothy);

9
data-store/src/test/java/org/lucares/pdb/datastore/internal/QueryCompletionIndexTest.java
View File

@@ -38,9 +38,12 @@ public class QueryCompletionIndexTest {
Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
final List<Tags> tags = Arrays.asList(//
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country", "Atlantis"), // A
Tags.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country", "ElDorado"), // B
Tags.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country", "Atlantis")// C
Tags.STRING_COMPRESSOR.createAndAddToDictionary("firstname", "John", "lastname", "Doe", "country",
"Atlantis"), // A
Tags.STRING_COMPRESSOR.createAndAddToDictionary("firstname", "Jane", "lastname", "Doe", "country",
"ElDorado"), // B
Tags.STRING_COMPRESSOR.createAndAddToDictionary("firstname", "John", "lastname", "Miller", "country",
"Atlantis")// C
);
final DateTimeRange dateRange = DateTimeRange.relativeMillis(1);