andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

introduce index clustering (part 1)

Browse Source 
In order to prevent files from getting too big and
make it easier to implement retention policies, we
are splitting all files into chunks. Each chunk
contains the data for a time interval (1 month per
default).
This first changeset introduces the ClusteredPersistentMap
that implements this for PersistentMap. It is used
for a couple (not all) of indices.
This commit is contained in:
Andreas Huber
2019-02-24 16:50:57 +01:00
parent 372a073b6d
commit 59aea1a15f
25 changed files with 863 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pdb-ui/src/main/java/org/lucares/pdbui/PdbController.java
View File

@@ -15,6 +15,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.StringUtils;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.Query;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.plot.api.AxisScale;
import org.lucares.pdb.plot.api.Limit;
@@ -228,7 +230,10 @@ public class PdbController implements HardcodedValues, PropertyKeys {
)
@ResponseBody
List<String> fields() {
final List<String> fields = db.getFields();
// TODO get date range from UI
// TODO time range must not be static
final DateTimeRange dateTimeRange = DateTimeRange.relativeYears(5);
final List<String> fields = db.getFields(dateTimeRange);
fields.sort(Collator.getInstance(Locale.ENGLISH));
@@ -244,7 +249,11 @@ public class PdbController implements HardcodedValues, PropertyKeys {
SortedSet<String> fields(@PathVariable(name = "fieldName") final String fieldName,
@RequestParam(name = "query") final String query) {
final SortedSet<String> fields = db.getFieldsValues(query, fieldName);
// TODO get date range from UI
// TODO time range must not be static
final DateTimeRange dateRange = DateTimeRange.relativeYears(5);
final Query q = new Query(query, dateRange);
final SortedSet<String> fields = db.getFieldsValues(q, fieldName);
return fields;
}

76
pdb-ui/src/main/java/org/lucares/pdbui/domain/DateRange.java Normal file
View File

@@ -0,0 +1,76 @@
package org.lucares.pdbui.domain;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import org.lucares.pdb.api.DateTimeRange;
public class DateRange {
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private String startDate;
private String endDate;
DateRange() {
super();
}
/**
*
* @param startDate date in format 'yyyy-MM-dd HH:mm:ss'
* @param endDate date in format 'yyyy-MM-dd HH:mm:ss'
*/
public DateRange(final String startDate, final String endDate) {
this.startDate = startDate;
this.endDate = endDate;
}
/**
*
* @return date in format 'yyyy-MM-dd HH:mm:ss'
*/
public String getStartDate() {
return startDate;
}
/**
*
* @param startDate date in format 'yyyy-MM-dd HH:mm:ss'
*/
public void setStartDate(final String startDate) {
this.startDate = startDate;
}
/**
*
* @return date in format 'yyyy-MM-dd HH:mm:ss'
*/
public String getEndDate() {
return endDate;
}
/**
*
* @param endDate date in format 'yyyy-MM-dd HH:mm:ss'
*/
public void setEndDate(final String endDate) {
this.endDate = endDate;
}
@Override
public String toString() {
return startDate + " - " + endDate;
}
public DateTimeRange toDateTimeRange() {
final OffsetDateTime start = LocalDateTime.parse(startDate, DATE_FORMAT).atOffset(ZoneOffset.UTC);
final OffsetDateTime end = LocalDateTime.parse(endDate, DATE_FORMAT).atOffset(ZoneOffset.UTC);
return new DateTimeRange(start, end);
}
}

17
pdb-ui/src/test/java/org/lucares/performance/db/ingestor/TcpIngestorTest.java
View File

@@ -18,6 +18,8 @@ import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.ThreadLocalRandom;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.DateTimeRange;
import org.lucares.pdb.api.Query;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdbui.TcpIngestor;
import org.lucares.performance.db.PdbExport;
@@ -79,10 +81,11 @@ public class TcpIngestorTest {
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get("host=" + host).singleGroup().flatMap();
final LongList result = db.get(new Query("host=" + host, DateTimeRange.ofDay(dateA))).singleGroup()
.flatMap();
Assert.assertEquals(result.size(), 4);
Assert.assertEquals(result.get(0), dateA.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli());
Assert.assertEquals(result.get(0), dateA.toInstant().toEpochMilli());
Assert.assertEquals(result.get(1), 1);
Assert.assertEquals(result.get(2), dateB.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli());
@@ -95,6 +98,7 @@ public class TcpIngestorTest {
final long dateA = Instant.now().toEpochMilli();
final long dateB = Instant.now().toEpochMilli() + 1;
final long dateC = Instant.now().toEpochMilli() - 1;
final DateTimeRange dateRange = DateTimeRange.relativeMinutes(1);
final String host = "someHost";
// 1. insert some data
@@ -133,7 +137,7 @@ public class TcpIngestorTest {
// 5. check that the data is correctly inserted
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get("host=" + host).singleGroup().flatMap();
final LongList result = db.get(new Query("host=" + host, dateRange)).singleGroup().flatMap();
Assert.assertEquals(result.size(), 6);
Assert.assertEquals(result.get(0), dateA);
@@ -151,6 +155,7 @@ public class TcpIngestorTest {
public void testIngestionThreadDoesNotDieOnErrors() throws Exception {
final OffsetDateTime dateA = OffsetDateTime.ofInstant(Instant.ofEpochMilli(-1), ZoneOffset.UTC);
final OffsetDateTime dateB = OffsetDateTime.now();
final DateTimeRange dateRange = new DateTimeRange(dateA, dateB);
final String host = "someHost";
try (TcpIngestor tcpIngestor = new TcpIngestor(dataDirectory)) {
@@ -185,7 +190,7 @@ public class TcpIngestorTest {
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get("host=" + host).singleGroup().flatMap();
final LongList result = db.get(new Query("host=" + host, dateRange)).singleGroup().flatMap();
Assert.assertEquals(result.size(), 4);
Assert.assertEquals(result.get(0), dateA.toInstant().truncatedTo(ChronoUnit.MILLIS).toEpochMilli());
@@ -212,6 +217,8 @@ public class TcpIngestorTest {
final ThreadLocalRandom rnd = ThreadLocalRandom.current();
final String host = "someHost";
final List<String> additionalTagValues = Arrays.asList("foo", "bar", "baz");
final DateTimeRange dateRange = new DateTimeRange(Instant.ofEpochMilli(-100000L).atOffset(ZoneOffset.UTC),
Instant.ofEpochMilli(10000000L).atOffset(ZoneOffset.UTC));
final LongList expected = new LongList();
@@ -245,7 +252,7 @@ public class TcpIngestorTest {
}
try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
final LongList result = db.get("host=" + host).singleGroup().flatMap();
final LongList result = db.get(new Query("host=" + host, dateRange)).singleGroup().flatMap();
Assert.assertEquals(LongPair.fromLongList(result), LongPair.fromLongList(expected));
}
}