andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

extract code from DateIndexExtension to LongToDateBucket

Browse Source 
Making it possible to reuse the code to sort timestamps
into date based buckets.
This commit is contained in:
Andreas Huber
2020-04-03 19:46:08 +02:00
parent 6288ed22bf
commit 75391f21ff
3 changed files with 206 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
pdb-utils/src/test/java/org/lucares/utils/LongToDateBucketTest.java Normal file
View File

@@ -0,0 +1,145 @@
package org.lucares.utils;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.DoubleSummaryStatistics;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
public class LongToDateBucketTest {
public static Stream<Arguments> provider() {
final List<Arguments> result = new ArrayList<>();
{
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801");
result.add(Arguments.of(start, end, expected));
}
{
final OffsetDateTime start = OffsetDateTime.of(2017, 11, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 02, 1, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201711", "201712", "201801", "201802");
result.add(Arguments.of(start, end, expected));
}
{
// check that adding one month to Jan 31 does not skip the February
final OffsetDateTime start = OffsetDateTime.of(2018, 1, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime end = OffsetDateTime.of(2018, 3, 31, 0, 0, 0, 0, ZoneOffset.UTC);
final Set<String> expected = Set.of("201801", "201802", "201803");
result.add(Arguments.of(start, end, expected));
}
return result.stream();
}
@ParameterizedTest
@MethodSource("provider")
public void test(final OffsetDateTime start, final OffsetDateTime end, final Set<String> expected) {
final Set<String> actual = new LongToDateBucket("yyyyMM").toDateIndexPrefix(start, end);
Assertions.assertEquals(expected, actual);
}
@Test
public void testDateToDateIndexPrefix() {
final long mid_201711 = OffsetDateTime.of(2017, 11, 23, 2, 2, 2, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant()
.toEpochMilli();
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
Assertions.assertEquals("201712", longToDateBucket.toDateIndexPrefix(mid_201712));
Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(min_201801));
Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(max_201801));
Assertions.assertEquals("201711", longToDateBucket.toDateIndexPrefix(mid_201711));
}
@Test
public void testDateRanges() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-2));
final OffsetDateTime min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(-8));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC)
.withOffsetSameInstant(ZoneOffset.ofHours(12));
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
final List<String> dateIndexPrefixesWithEmptyCache = longToDateBucket.toPartitionIds(mid_201712, min_201802);
Assertions.assertEquals(Arrays.asList("201712", "201801", "201802"), dateIndexPrefixesWithEmptyCache);
final List<String> dateIndexPrefixesWithFilledCache = longToDateBucket.toPartitionIds(mid_201712, min_201801);
Assertions.assertEquals(Arrays.asList("201712", "201801"), dateIndexPrefixesWithFilledCache);
final List<String> dateIndexPrefixesOneMonth = longToDateBucket.toPartitionIds(mid_201712, mid_201712);
Assertions.assertEquals(Arrays.asList("201712"), dateIndexPrefixesOneMonth);
}
@Test
public void testDateRangeToEpochMilli() {
final OffsetDateTime mid_201712 = OffsetDateTime.of(2017, 12, 7, 1, 1, 1, 0, ZoneOffset.ofHours(3));
final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 15, 0, 0, 0, 0, ZoneOffset.ofHours(7));
final long exp_201712 = OffsetDateTime.of(2017, 12, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
final List<Long> dateIndexEpochMillis = longToDateBucket.toDateIndexEpochMillis(mid_201712, min_201802);
Assertions.assertEquals(Arrays.asList(exp_201712, exp_201801, exp_201802), dateIndexEpochMillis);
}
@Test
public void testPerformance() {
final long min = OffsetDateTime.of(2010, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long mid = OffsetDateTime.of(2020, 6, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final long max = OffsetDateTime.of(2030, 12, 31, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli();
final int iterations = 1_000_000;
final int factor = 1;
final int warmup = 20 * factor;
final int rounds = warmup + 20;
final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM");
// fill the cache
for (long i = min; i < max; i += 3600 * 24 * 28) {
longToDateBucket.toPartitionId(i);
}
final List<Double> measurements = new ArrayList<>();
for (int r = 0; r < rounds; r++) {
final long start = System.nanoTime();
for (int i = 0; i < iterations; i++) {
longToDateBucket.toPartitionId(mid);
}
final double duration = (System.nanoTime() - start) / 1_000_000.0;
System.out.println("duration: " + duration + "ms");
measurements.add(duration);
}
final DoubleSummaryStatistics stats = measurements.subList(warmup, rounds).stream().mapToDouble(d -> factor * d)
.summaryStatistics();
System.out.println(stats);
}
}