add reindex method to PersistentMap

block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java

@@ -1,8 +1,12 @@
 package org.lucares.pdb.map;
 
+import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -10,6 +14,7 @@ import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Stack;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Function;
 
 import org.lucares.collections.LongList;
@@ -52,7 +57,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
         public byte[] getEmptyValue();
     }
 
-    private static final class StringCoder implements EncoderDecoder<String> {
+    public static final class StringCoder implements EncoderDecoder<String> {
 
         @Override
         public byte[] encode(final String object) {
@@ -70,7 +75,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
         }
     }
 
-    private static final class LongCoder implements EncoderDecoder<Long> {
+    public static final class LongCoder implements EncoderDecoder<Long> {
 
         @Override
         public byte[] encode(final Long object) {
@@ -88,7 +93,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
         }
     }
 
-    private static final class UUIDCoder implements EncoderDecoder<UUID> {
+    public static final class UUIDCoder implements EncoderDecoder<UUID> {
 
         @Override
         public byte[] encode(final UUID uuid) {
@@ -143,7 +148,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
     static final int BLOCK_SIZE = 4096;
     static final long NODE_OFFSET_TO_ROOT_NODE = 8;
 
-    private final DiskStorage diskStore;
+    private DiskStorage diskStore;
 
     private int maxEntriesInNode = Integer.MAX_VALUE;
 
@@ -158,13 +163,20 @@ public class PersistentMap<K, V> implements AutoCloseable {
     // guarded by: this
     private volatile long nodeOffsetOfRootNode = -1;
 
+    private final Path path;
+
     public PersistentMap(final Path path, final Path storageBasePath, final EncoderDecoder<K> keyEncoder,
             final EncoderDecoder<V> valueEncoder) {
+        this.path = path;
         this.diskStore = new DiskStorage(path, storageBasePath);
         this.keyEncoder = keyEncoder;
         this.valueEncoder = valueEncoder;
         initIfNew();
 
+        readOffsetOfRootNode();
+    }
+
+    private void readOffsetOfRootNode() {
         final DiskBlock diskBlock = diskStore.getDiskBlock(NODE_OFFSET_TO_ROOT_NODE, diskStore.minAllocationSize());
         nodeOffsetOfRootNode = diskBlock.getByteBuffer().getLong(0);
     }
@@ -237,7 +249,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
     }
 
     private V getFromValueCache(final byte[] encodedKey, final K key) {
-        LOGGER.info("valueCache hit rate: {} when getting key: {}", valueCache.cacheHitRate(), key);
+        LOGGER.trace("valueCache hit rate: {} when getting key: {}", valueCache.cacheHitRate(), key);
         return valueCache.get(new ByteArrayKey(encodedKey));
     }
 
@@ -460,6 +472,41 @@ public class PersistentMap<K, V> implements AutoCloseable {
         iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor);
     }
 
+    public synchronized void reindex() throws IOException {
+        final long start = System.nanoTime();
+        final AtomicLong countValues = new AtomicLong();
+        LOGGER.info("start reindexing file: {}", path);
+        final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp");
+
+        try (PersistentMap<K, V> newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) {
+            final long rootNodeOffset = readNodeOffsetOfRootNode();
+            final byte[] encodedKeyPrefix = new byte[0];
+            iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, (k, v) -> {
+                newMap.putValue(k, v);
+                final long count = countValues.incrementAndGet();
+                if (count % 100000 == 0) {
+                    LOGGER.info("written {} values", count);
+                }
+            });
+        }
+
+        diskStore.close();
+        valueCache.clear();
+        nodeCache.clear();
+
+        final Path backupFile = path.getParent().resolve(path.getFileName() + "."
+                + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").format(OffsetDateTime.now()) + ".backup");
+        Files.move(path, backupFile);
+        Files.move(newFile, path);
+
+        this.diskStore = new DiskStorage(path, null);
+        readOffsetOfRootNode();
+        final double durationInMs = (System.nanoTime() - start) / 1_000_000.0;
+        final double valuesPerSecond = countValues.get() / (durationInMs / 1000);
+        LOGGER.info("done reindexing, took {} ms, {} values, {} values/s", (int) Math.ceil(durationInMs),
+                countValues.get(), valuesPerSecond);
+    }
+
     private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor<K, V> visitor) {
         final PersistentMapDiskNode node = getNode(nodeOffest);