extract code from DateIndexExtension to LongToDateBucket
Making it possible to reuse the code to sort timestamps into date based buckets.
This commit is contained in:
173
pdb-utils/src/main/java/org/lucares/utils/LongToDateBucket.java
Normal file
173
pdb-utils/src/main/java/org/lucares/utils/LongToDateBucket.java
Normal file
@@ -0,0 +1,173 @@
|
||||
package org.lucares.utils;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentNavigableMap;
|
||||
import java.util.concurrent.ConcurrentSkipListMap;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
public class LongToDateBucket {
|
||||
|
||||
private static final class DatePrefixAndRange {
|
||||
private final String datePrefix;
|
||||
private final long minEpochMilli;
|
||||
private final long maxEpochMilli;
|
||||
|
||||
public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) {
|
||||
super();
|
||||
this.datePrefix = datePrefix;
|
||||
this.minEpochMilli = minEpochMilli;
|
||||
this.maxEpochMilli = maxEpochMilli;
|
||||
}
|
||||
|
||||
public String getDatePrefix() {
|
||||
return datePrefix;
|
||||
}
|
||||
|
||||
public long getMinEpochMilli() {
|
||||
return minEpochMilli;
|
||||
}
|
||||
|
||||
public long getMaxEpochMilli() {
|
||||
return maxEpochMilli;
|
||||
}
|
||||
|
||||
public boolean contains(final long epochMilli) {
|
||||
return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This date pattern defines the resolution of the date index
|
||||
*/
|
||||
private final DateTimeFormatter datePattern;
|
||||
|
||||
// visible for test
|
||||
final ConcurrentNavigableMap<Long, DatePrefixAndRange> datePrefixCache = new ConcurrentSkipListMap<>();
|
||||
|
||||
private final AtomicReference<DatePrefixAndRange> lastAccessed = new AtomicReference<>(null);
|
||||
|
||||
public LongToDateBucket(final String dateFormatPattern) {
|
||||
this.datePattern = DateTimeFormatter.ofPattern(dateFormatPattern);
|
||||
}
|
||||
|
||||
public Set<String> toDateIndexPrefix(final OffsetDateTime start, final OffsetDateTime end) {
|
||||
final Set<String> result = new TreeSet<>();
|
||||
|
||||
OffsetDateTime current = start;
|
||||
while (current.isBefore(end)) {
|
||||
|
||||
result.add(toDateIndexPrefix(current));
|
||||
current = current.plusMonths(1);
|
||||
|
||||
}
|
||||
result.add(toDateIndexPrefix(end));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public String toDateIndexPrefix(final OffsetDateTime time) {
|
||||
return time.format(datePattern);
|
||||
}
|
||||
|
||||
public String toPartitionId(final long epochMilli) {
|
||||
String result;
|
||||
final DatePrefixAndRange lastAccessed = this.lastAccessed.get();
|
||||
if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli
|
||||
&& lastAccessed.getMaxEpochMilli() >= epochMilli) {
|
||||
result = lastAccessed.getDatePrefix();
|
||||
} else {
|
||||
final Entry<Long, DatePrefixAndRange> value = datePrefixCache.floorEntry(epochMilli);
|
||||
|
||||
if (value == null || !value.getValue().contains(epochMilli)) {
|
||||
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
|
||||
datePrefixCache.put(newValue.getMinEpochMilli(), newValue);
|
||||
result = newValue.getDatePrefix();
|
||||
this.lastAccessed.set(newValue);
|
||||
} else {
|
||||
result = value.getValue().getDatePrefix();
|
||||
this.lastAccessed.set(value.getValue());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public String toDateIndexPrefix(final long epochMilli) {
|
||||
|
||||
final Entry<Long, DatePrefixAndRange> value = datePrefixCache.floorEntry(epochMilli);
|
||||
|
||||
String result;
|
||||
if (value == null || !value.getValue().contains(epochMilli)) {
|
||||
final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli);
|
||||
datePrefixCache.put(newValue.getMinEpochMilli(), newValue);
|
||||
result = newValue.getDatePrefix();
|
||||
} else {
|
||||
result = value.getValue().getDatePrefix();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* only for tests, use toPartitionIds(final DateTimeRange dateRange,final
|
||||
* Collection<? extends PartitionId> availablePartitionIds) instead
|
||||
*
|
||||
* @param dateRange
|
||||
* @return
|
||||
*/
|
||||
public List<String> toPartitionIds(final OffsetDateTime start, final OffsetDateTime end) {
|
||||
final List<String> result = new ArrayList<>();
|
||||
|
||||
OffsetDateTime current = start;
|
||||
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
|
||||
.withSecond(0).withNano(0);
|
||||
|
||||
while (!current.isAfter(end)) {
|
||||
final String id = toDateIndexPrefix(current);
|
||||
result.add(id);
|
||||
current = current.plusMonths(1);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) {
|
||||
final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC);
|
||||
final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
|
||||
final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1);
|
||||
|
||||
final String datePrefix = date.format(datePattern);
|
||||
final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli();
|
||||
final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli();
|
||||
|
||||
return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli);
|
||||
}
|
||||
|
||||
public List<Long> toDateIndexEpochMillis(final OffsetDateTime start, final OffsetDateTime end) {
|
||||
final List<Long> result = new ArrayList<>();
|
||||
|
||||
OffsetDateTime current = start;
|
||||
current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0)
|
||||
.withSecond(0).withNano(0);
|
||||
|
||||
while (!current.isAfter(end)) {
|
||||
result.add(current.toInstant().toEpochMilli());
|
||||
current = current.plusMonths(1);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package org.lucares.utils;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.DoubleSummaryStatistics;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
|
||||
public class LongToDateBucketTest {
|
||||
|
||||
public static Stream<Arguments> provider() {
|
||||
|
||||
final List<Arguments> result = new ArrayList<>();
|
||||
|
||||
{
|
||||
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final Set<String> expected = Set.of("201801");
|
||||
result.add(Arguments.of(start, end, expected));
|
||||
}
|
||||
{
|
||||
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
|
||||
result.add(Arguments.of(start, end, expected));
|
||||
}
|
||||
{
|
||||
// check that adding one month to Jan 31 does not skip the February
|
||||
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
|
||||
final Set<String> expected = Set.of("201801", "201802", "201803");
|
||||
result.add(Arguments.of(start, end, expected));
|
||||
}
|
||||
|
||||
return result.stream();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@MethodSource("provider")
|
||||
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
|
||||
|
||||
final Set<String> actual = new LongToDateBucket("yyyyMM").toDateIndexPrefix(start, end);
|
||||
|
||||
Assertions.assertEquals(expected, actual);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDateToDateIndexPrefix() {
|
||||
|
||||
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
|
||||
.toEpochMilli();
|
||||
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
|
||||
|
||||
Assertions.assertEquals("201712", longToDateBucket.toDateIndexPrefix(mid_201712));
|
||||
Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(min_201801));
|
||||
Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(max_201801));
|
||||
Assertions.assertEquals("201711", longToDateBucket.toDateIndexPrefix(mid_201711));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDateRanges() {
|
||||
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
|
||||
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
|
||||
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
|
||||
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
|
||||
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
|
||||
.withOffsetSameInstant(ZoneOffset.ofHours(12));
|
||||
|
||||
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
|
||||
|
||||
final List<String> dateIndexPrefixesWithEmptyCache = longToDateBucket.toPartitionIds(mid_201712, min_201802);
|
||||
Assertions.assertEquals(Arrays.asList("201712", "201801", "201802"), dateIndexPrefixesWithEmptyCache);
|
||||
|
||||
final List<String> dateIndexPrefixesWithFilledCache = longToDateBucket.toPartitionIds(mid_201712, min_201801);
|
||||
Assertions.assertEquals(Arrays.asList("201712", "201801"), dateIndexPrefixesWithFilledCache);
|
||||
|
||||
final List<String> dateIndexPrefixesOneMonth = longToDateBucket.toPartitionIds(mid_201712, mid_201712);
|
||||
Assertions.assertEquals(Arrays.asList("201712"), dateIndexPrefixesOneMonth);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDateRangeToEpochMilli() {
|
||||
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
|
||||
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
|
||||
|
||||
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
|
||||
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
|
||||
|
||||
final List<Long> dateIndexEpochMillis = longToDateBucket.toDateIndexEpochMillis(mid_201712, min_201802);
|
||||
Assertions.assertEquals(Arrays.asList(exp_201712, exp_201801, exp_201802), dateIndexEpochMillis);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPerformance() {
|
||||
|
||||
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
|
||||
|
||||
final int iterations = 1_000_000;
|
||||
final int factor = 1;
|
||||
final int warmup = 20 * factor;
|
||||
final int rounds = warmup + 20;
|
||||
|
||||
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
|
||||
|
||||
// fill the cache
|
||||
for (long i = min; i < max; i += 3600 * 24 * 28) {
|
||||
longToDateBucket.toPartitionId(i);
|
||||
}
|
||||
|
||||
final List<Double> measurements = new ArrayList<>();
|
||||
|
||||
for (int r = 0; r < rounds; r++) {
|
||||
|
||||
final long start = System.nanoTime();
|
||||
for (int i = 0; i < iterations; i++) {
|
||||
longToDateBucket.toPartitionId(mid);
|
||||
}
|
||||
final double duration = (System.nanoTime() - start) / 1_000_000.0;
|
||||
System.out.println("duration: " + duration + "ms");
|
||||
measurements.add(duration);
|
||||
}
|
||||
|
||||
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
|
||||
.summaryStatistics();
|
||||
System.out.println(stats);
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user