andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

prepare the addition of a date index

Browse Source
This commit is contained in:
Andreas Huber
2018-09-28 19:07:01 +02:00
parent 1d88c8dfd7
commit 24fcfd7763
11 changed files with 232 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
data-store/src/main/java/org/lucares/pdb/datastore/DateTimeRange.java Normal file
View File

@@ -0,0 +1,29 @@
package org.lucares.pdb.datastore;
import java.time.OffsetDateTime;
public class DateTimeRange {
public static final DateTimeRange MAX = new DateTimeRange(OffsetDateTime.MIN, OffsetDateTime.MAX);
private final OffsetDateTime start;
private final OffsetDateTime end;
public DateTimeRange(final OffsetDateTime start, final OffsetDateTime end) {
this.start = start;
this.end = end;
}
public OffsetDateTime getStart() {
return start;
}
public OffsetDateTime getEnd() {
return end;
}
@Override
public String toString() {
return start + "-" + end;
}
}

109
data-store/src/main/java/org/lucares/pdb/datastore/lang/DateIndexExtension.java Normal file
View File

@@ -0,0 +1,109 @@
package org.lucares.pdb.datastore.lang;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import org.lucares.pdb.datastore.DateTimeRange;
public class DateIndexExtension {
/**
* This date pattern defines the resolution of the date index
*/
private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM");
// visible for test
static final ConcurrentNavigableMap<Long, DatePrefixAndRange> DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>();
static Set<String> toDateIndexPrefix(final DateTimeRange dateRange) {
final Set<String> result = new TreeSet<>();
if (Objects.equals(dateRange, DateTimeRange.MAX)) {
result.add("*");
} else {
OffsetDateTime current = dateRange.getStart();
while (current.isBefore(dateRange.getEnd())) {
result.add(toDateIndexPrefix(current));
current = current.plusMonths(1);
}
result.add(toDateIndexPrefix(dateRange.getEnd()));
}
return result;
}
static String toDateIndexPrefix(final OffsetDateTime time) {
return time.format(DATE_PATTERN);
}
public static String toDateIndexPrefix(final long epochMilli) {
final Entry<Long, DatePrefixAndRange> value = DATE_PREFIX_CACHE.floorEntry(epochMilli);
String result;
if (value == null || !value.getValue().contains(epochMilli)) {
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue);
result = newValue.getDatePrefix();
} else {
result = value.getValue().getDatePrefix();
}
return result;
}
private static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
final String datePrefix = date.format(DATE_PATTERN);
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
}
}
class DatePrefixAndRange {
private final String datePrefix;
private final long minEpochMilli;
private final long maxEpochMilli;
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
super();
this.datePrefix = datePrefix;
this.minEpochMilli = minEpochMilli;
this.maxEpochMilli = maxEpochMilli;
}
public String getDatePrefix() {
return datePrefix;
}
public long getMinEpochMilli() {
return minEpochMilli;
}
public long getMaxEpochMilli() {
return maxEpochMilli;
}
public boolean contains(final long epochMilli) {
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
}
@Override
public String toString() {
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
}
}

18
data-store/src/main/java/org/lucares/pdb/datastore/lang/Expression.java
View File

@@ -1,6 +1,7 @@
package org.lucares.pdb.datastore.lang;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.lucares.utils.CollectionUtils;
@@ -190,6 +191,19 @@ abstract public class Expression {
return true;
}
public static Expression create(final List<Expression> or) {
if (or.size() == 1) {
return or.get(0);
} else {
Or result = new Or(or.get(0), or.get(1));
for (int i = 2; i < or.size(); i++) {
result = new Or(result, or.get(i));
}
return result;
}
}
}
static class And extends Expression {
@@ -465,6 +479,10 @@ abstract public class Expression {
private final String property;
private final List<String> values;
public InExpression(final String property, final String value) {
this(property, Arrays.asList(value));
}
public InExpression(final String property, final List<String> values) {
this.property = property;
this.values = values;

1
data-store/src/main/java/org/lucares/pdb/datastore/lang/ExpressionToDocIdVisitor.java
View File

@@ -49,7 +49,6 @@ public class ExpressionToDocIdVisitor extends ExpressionVisitor<IntList> {
}
private static final Map<String, IntList> EMPTY_VALUES = Collections.emptyMap();
private static final IntList EMPTY_DOC_IDS = new IntList();
private final Map<String, Map<String, IntList>> keyToValueToDocId;
private final AllDocIds allDocIds;

69
data-store/src/test/java/org/lucares/pdb/datastore/lang/DateIndexExtensionTest.java Normal file
View File

@@ -0,0 +1,69 @@
package org.lucares.pdb.datastore.lang;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.lucares.pdb.datastore.DateTimeRange;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@Test
public class DateIndexExtensionTest {
@DataProvider
public Object[][] provider() {
final List<Object[]> result = new ArrayList<>();
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(new Object[] { start, end, expected });
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(new Object[] { start, end, expected });
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(new Object[] { start, end, expected });
}
return result.toArray(new Object[0][]);
}
@Test(dataProvider = "provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
final DateTimeRange dateRange = new DateTimeRange(start, end);
final Set<String> actual = DateIndexExtension.toDateIndexPrefix(dateRange);
Assert.assertEquals(actual, expected);
}
public void testDateToDateIndexPrefix() {
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201712), "201712");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(min_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(max_201801), "201801");
Assert.assertEquals(DateIndexExtension.toDateIndexPrefix(mid_201711), "201711");
System.out.println(DateIndexExtension.DATE_PREFIX_CACHE);
}
}