diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java index 8f498b7..43c13ef 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DateIndexExtension.java @@ -1,89 +1,28 @@ package org.lucares.pdb.datastore.internal; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map.Entry; import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicReference; import org.lucares.pdb.api.DateTimeRange; +import org.lucares.utils.LongToDateBucket; public class DateIndexExtension { - /** - * This date pattern defines the resolution of the date index - */ - private static final DateTimeFormatter DATE_PATTERN = DateTimeFormatter.ofPattern("yyyyMM"); - - // visible for test - static final ConcurrentNavigableMap DATE_PREFIX_CACHE = new ConcurrentSkipListMap<>(); - - private static final AtomicReference LAST_ACCESSED = new AtomicReference<>(null); + private static final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM"); static Set toDateIndexPrefix(final DateTimeRange dateRange) { - final Set result = new TreeSet<>(); - - OffsetDateTime current = dateRange.getStart(); - while (current.isBefore(dateRange.getEnd())) { - - result.add(toDateIndexPrefix(current)); - current = current.plusMonths(1); - - } - result.add(toDateIndexPrefix(dateRange.getEnd())); - - return result; - } - - static String toDateIndexPrefix(final OffsetDateTime time) { - return time.format(DATE_PATTERN); + return longToDateBucket.toDateIndexPrefix(dateRange.getStart(), dateRange.getEnd()); } public static ParititionId toPartitionId(final long epochMilli) { - String result; - final DatePrefixAndRange lastAccessed = LAST_ACCESSED.get(); - if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli - && lastAccessed.getMaxEpochMilli() >= epochMilli) { - result = lastAccessed.getDatePrefix(); - } else { - final Entry value = DATE_PREFIX_CACHE.floorEntry(epochMilli); - - if (value == null || !value.getValue().contains(epochMilli)) { - final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); - DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue); - result = newValue.getDatePrefix(); - LAST_ACCESSED.set(newValue); - } else { - result = value.getValue().getDatePrefix(); - LAST_ACCESSED.set(value.getValue()); - } - } - return new ParititionId(result); + return new ParititionId(longToDateBucket.toPartitionId(epochMilli)); } public static String toDateIndexPrefix(final long epochMilli) { - - final Entry value = DATE_PREFIX_CACHE.floorEntry(epochMilli); - - String result; - if (value == null || !value.getValue().contains(epochMilli)) { - final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); - DATE_PREFIX_CACHE.put(newValue.getMinEpochMilli(), newValue); - result = newValue.getDatePrefix(); - } else { - result = value.getValue().getDatePrefix(); - } - - return result; + return longToDateBucket.toDateIndexPrefix(epochMilli); } /** @@ -94,20 +33,12 @@ public class DateIndexExtension { * @return */ static List toPartitionIds(final DateTimeRange dateRange) { + final List partitionIds = longToDateBucket.toPartitionIds(dateRange.getStart(), dateRange.getEnd()); + final List result = new ArrayList<>(); - - OffsetDateTime current = dateRange.getStart(); - final OffsetDateTime end = dateRange.getEnd(); - current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) - .withSecond(0).withNano(0); - - while (!current.isAfter(end)) { - final String id = current.format(DATE_PATTERN); - final ParititionId partitionId = new ParititionId(id); - result.add(partitionId); - current = current.plusMonths(1); + for (final String partitionId : partitionIds) { + result.add(new ParititionId(partitionId)); } - return result; } @@ -127,70 +58,8 @@ public class DateIndexExtension { return result; } - public static DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) { - final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); - final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0); - final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1); - - final String datePrefix = date.format(DATE_PATTERN); - final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli(); - final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli(); - - return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli); - } - - public static List toDateIndexEpochMillis(final DateTimeRange dateRange) { - final List result = new ArrayList<>(); - - OffsetDateTime current = dateRange.getStart(); - final OffsetDateTime end = dateRange.getEnd(); - current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) - .withSecond(0).withNano(0); - - while (!current.isAfter(end)) { - result.add(current.toInstant().toEpochMilli()); - current = current.plusMonths(1); - } - - return result; - } - public static ParititionId now() { return toPartitionId(System.currentTimeMillis()); } } - -class DatePrefixAndRange { - private final String datePrefix; - private final long minEpochMilli; - private final long maxEpochMilli; - - public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) { - super(); - this.datePrefix = datePrefix; - this.minEpochMilli = minEpochMilli; - this.maxEpochMilli = maxEpochMilli; - } - - public String getDatePrefix() { - return datePrefix; - } - - public long getMinEpochMilli() { - return minEpochMilli; - } - - public long getMaxEpochMilli() { - return maxEpochMilli; - } - - public boolean contains(final long epochMilli) { - return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli; - } - - @Override - public String toString() { - return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")"; - } -} \ No newline at end of file diff --git a/pdb-utils/src/main/java/org/lucares/utils/LongToDateBucket.java b/pdb-utils/src/main/java/org/lucares/utils/LongToDateBucket.java new file mode 100644 index 0000000..13e9942 --- /dev/null +++ b/pdb-utils/src/main/java/org/lucares/utils/LongToDateBucket.java @@ -0,0 +1,173 @@ +package org.lucares.utils; + +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.atomic.AtomicReference; + +public class LongToDateBucket { + + private static final class DatePrefixAndRange { + private final String datePrefix; + private final long minEpochMilli; + private final long maxEpochMilli; + + public DatePrefixAndRange(final String datePrefix, final long minEpochMilli, final long maxEpochMilli) { + super(); + this.datePrefix = datePrefix; + this.minEpochMilli = minEpochMilli; + this.maxEpochMilli = maxEpochMilli; + } + + public String getDatePrefix() { + return datePrefix; + } + + public long getMinEpochMilli() { + return minEpochMilli; + } + + public long getMaxEpochMilli() { + return maxEpochMilli; + } + + public boolean contains(final long epochMilli) { + return minEpochMilli <= epochMilli && epochMilli <= maxEpochMilli; + } + + @Override + public String toString() { + return datePrefix + " (" + minEpochMilli + " - " + maxEpochMilli + ")"; + } + } + + /** + * This date pattern defines the resolution of the date index + */ + private final DateTimeFormatter datePattern; + + // visible for test + final ConcurrentNavigableMap datePrefixCache = new ConcurrentSkipListMap<>(); + + private final AtomicReference lastAccessed = new AtomicReference<>(null); + + public LongToDateBucket(final String dateFormatPattern) { + this.datePattern = DateTimeFormatter.ofPattern(dateFormatPattern); + } + + public Set toDateIndexPrefix(final OffsetDateTime start, final OffsetDateTime end) { + final Set result = new TreeSet<>(); + + OffsetDateTime current = start; + while (current.isBefore(end)) { + + result.add(toDateIndexPrefix(current)); + current = current.plusMonths(1); + + } + result.add(toDateIndexPrefix(end)); + + return result; + } + + public String toDateIndexPrefix(final OffsetDateTime time) { + return time.format(datePattern); + } + + public String toPartitionId(final long epochMilli) { + String result; + final DatePrefixAndRange lastAccessed = this.lastAccessed.get(); + if (lastAccessed != null && lastAccessed.getMinEpochMilli() <= epochMilli + && lastAccessed.getMaxEpochMilli() >= epochMilli) { + result = lastAccessed.getDatePrefix(); + } else { + final Entry value = datePrefixCache.floorEntry(epochMilli); + + if (value == null || !value.getValue().contains(epochMilli)) { + final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); + datePrefixCache.put(newValue.getMinEpochMilli(), newValue); + result = newValue.getDatePrefix(); + this.lastAccessed.set(newValue); + } else { + result = value.getValue().getDatePrefix(); + this.lastAccessed.set(value.getValue()); + } + } + return result; + } + + public String toDateIndexPrefix(final long epochMilli) { + + final Entry value = datePrefixCache.floorEntry(epochMilli); + + String result; + if (value == null || !value.getValue().contains(epochMilli)) { + final DatePrefixAndRange newValue = toDatePrefixAndRange(epochMilli); + datePrefixCache.put(newValue.getMinEpochMilli(), newValue); + result = newValue.getDatePrefix(); + } else { + result = value.getValue().getDatePrefix(); + } + + return result; + } + + /** + * only for tests, use toPartitionIds(final DateTimeRange dateRange,final + * Collection availablePartitionIds) instead + * + * @param dateRange + * @return + */ + public List toPartitionIds(final OffsetDateTime start, final OffsetDateTime end) { + final List result = new ArrayList<>(); + + OffsetDateTime current = start; + current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) + .withSecond(0).withNano(0); + + while (!current.isAfter(end)) { + final String id = toDateIndexPrefix(current); + result.add(id); + current = current.plusMonths(1); + } + + return result; + } + + private DatePrefixAndRange toDatePrefixAndRange(final long epochMilli) { + final OffsetDateTime date = Instant.ofEpochMilli(epochMilli).atOffset(ZoneOffset.UTC); + final OffsetDateTime beginOfMonth = date.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0); + final OffsetDateTime endOfMonth = beginOfMonth.plusMonths(1).minusNanos(1); + + final String datePrefix = date.format(datePattern); + final long minEpochMilli = beginOfMonth.toInstant().toEpochMilli(); + final long maxEpochMilli = endOfMonth.toInstant().toEpochMilli(); + + return new DatePrefixAndRange(datePrefix, minEpochMilli, maxEpochMilli); + } + + public List toDateIndexEpochMillis(final OffsetDateTime start, final OffsetDateTime end) { + final List result = new ArrayList<>(); + + OffsetDateTime current = start; + current = current.withOffsetSameInstant(ZoneOffset.UTC).withDayOfMonth(1).withHour(0).withMinute(0) + .withSecond(0).withNano(0); + + while (!current.isAfter(end)) { + result.add(current.toInstant().toEpochMilli()); + current = current.plusMonths(1); + } + + return result; + } + +} diff --git a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java b/pdb-utils/src/test/java/org/lucares/utils/LongToDateBucketTest.java similarity index 71% rename from data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java rename to pdb-utils/src/test/java/org/lucares/utils/LongToDateBucketTest.java index 568c5de..66192db 100644 --- a/data-store/src/test/java/org/lucares/pdb/datastore/internal/DateIndexExtensionTest.java +++ b/pdb-utils/src/test/java/org/lucares/utils/LongToDateBucketTest.java @@ -1,4 +1,4 @@ -package org.lucares.pdb.datastore.internal; +package org.lucares.utils; import java.time.OffsetDateTime; import java.time.ZoneOffset; @@ -9,14 +9,13 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.lucares.pdb.api.DateTimeRange; -import org.junit.jupiter.api.Assertions; -public class DateIndexExtensionTest { +public class LongToDateBucketTest { public static Stream provider() { @@ -49,9 +48,7 @@ public class DateIndexExtensionTest { @MethodSource("provider") public void test(final OffsetDateTime start, final OffsetDateTime end, final Set expected) { - final DateTimeRange dateRange = new DateTimeRange(start, end); - - final Set actual = DateIndexExtension.toDateIndexPrefix(dateRange); + final Set actual = new LongToDateBucket("yyyyMM").toDateIndexPrefix(start, end); Assertions.assertEquals(expected, actual); } @@ -64,11 +61,12 @@ public class DateIndexExtensionTest { final long min_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long max_201801 = OffsetDateTime.of(2018, 1, 31, 23, 59, 59, 999_999_999, ZoneOffset.UTC).toInstant() .toEpochMilli(); + final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM"); - Assertions.assertEquals("201712", DateIndexExtension.toDateIndexPrefix(mid_201712)); - Assertions.assertEquals("201801", DateIndexExtension.toDateIndexPrefix(min_201801)); - Assertions.assertEquals("201801", DateIndexExtension.toDateIndexPrefix(max_201801)); - Assertions.assertEquals("201711", DateIndexExtension.toDateIndexPrefix(mid_201711)); + Assertions.assertEquals("201712", longToDateBucket.toDateIndexPrefix(mid_201712)); + Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(min_201801)); + Assertions.assertEquals("201801", longToDateBucket.toDateIndexPrefix(max_201801)); + Assertions.assertEquals("201711", longToDateBucket.toDateIndexPrefix(mid_201711)); } @Test @@ -80,22 +78,16 @@ public class DateIndexExtensionTest { final OffsetDateTime min_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC) .withOffsetSameInstant(ZoneOffset.ofHours(12)); - final DateTimeRange range_201712_201802 = new DateTimeRange(mid_201712, min_201802); - final DateTimeRange range_201712_201801 = new DateTimeRange(mid_201712, min_201801); - final DateTimeRange range_201712_201712 = new DateTimeRange(mid_201712, mid_201712); + final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM"); - final List dateIndexPrefixesWithEmptyCache = DateIndexExtension - .toPartitionIds(range_201712_201802); - Assertions.assertEquals(Arrays.asList(new ParititionId("201712"), new ParititionId("201801"), new ParititionId("201802")), - dateIndexPrefixesWithEmptyCache); + final List dateIndexPrefixesWithEmptyCache = longToDateBucket.toPartitionIds(mid_201712, min_201802); + Assertions.assertEquals(Arrays.asList("201712", "201801", "201802"), dateIndexPrefixesWithEmptyCache); - final List dateIndexPrefixesWithFilledCache = DateIndexExtension - .toPartitionIds(range_201712_201801); - Assertions.assertEquals(Arrays.asList(new ParititionId("201712"), new ParititionId("201801")), - dateIndexPrefixesWithFilledCache); + final List dateIndexPrefixesWithFilledCache = longToDateBucket.toPartitionIds(mid_201712, min_201801); + Assertions.assertEquals(Arrays.asList("201712", "201801"), dateIndexPrefixesWithFilledCache); - final List dateIndexPrefixesOneMonth = DateIndexExtension.toPartitionIds(range_201712_201712); - Assertions.assertEquals(Arrays.asList(new ParititionId("201712")), dateIndexPrefixesOneMonth); + final List dateIndexPrefixesOneMonth = longToDateBucket.toPartitionIds(mid_201712, mid_201712); + Assertions.assertEquals(Arrays.asList("201712"), dateIndexPrefixesOneMonth); } @Test @@ -107,8 +99,9 @@ public class DateIndexExtensionTest { final long exp_201801 = OffsetDateTime.of(2018, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); final long exp_201802 = OffsetDateTime.of(2018, 2, 1, 0, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); - final List dateIndexEpochMillis = DateIndexExtension - .toDateIndexEpochMillis(new DateTimeRange(mid_201712, min_201802)); + final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM"); + + final List dateIndexEpochMillis = longToDateBucket.toDateIndexEpochMillis(mid_201712, min_201802); Assertions.assertEquals(Arrays.asList(exp_201712, exp_201801, exp_201802), dateIndexEpochMillis); } @@ -124,10 +117,11 @@ public class DateIndexExtensionTest { final int warmup = 20 * factor; final int rounds = warmup + 20; + final LongToDateBucket longToDateBucket = new LongToDateBucket("yyyyMM"); + // fill the cache - DateIndexExtension.DATE_PREFIX_CACHE.clear(); for (long i = min; i < max; i += 3600 * 24 * 28) { - DateIndexExtension.toPartitionId(i); + longToDateBucket.toPartitionId(i); } final List measurements = new ArrayList<>(); @@ -136,7 +130,7 @@ public class DateIndexExtensionTest { final long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - DateIndexExtension.toPartitionId(mid); + longToDateBucket.toPartitionId(mid); } final double duration = (System.nanoTime() - start) / 1_000_000.0; System.out.println("duration: " + duration + "ms"); @@ -147,4 +141,5 @@ public class DateIndexExtensionTest { .summaryStatistics(); System.out.println(stats); } + }