make it possible to reindex the persistent maps via system property

block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java

@@ -188,6 +188,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
 
     private long version;
 
+    private boolean isNew;
+
     /**
      * 
      * @param path            file for the index, must be child of storageBasePath
@@ -218,6 +220,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
 
     private void initIfNew() {
         if (diskStore.size() < BLOCK_SIZE) {
+            isNew = true;
             final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
             Preconditions.checkEqual(nodeOffsetToRootNode, OFFSET_META_DATA,
                     "The offset of the pointer to the root node must be at a well known location. "
@@ -582,7 +585,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
         final long start = System.nanoTime();
         final AtomicLong countValues = new AtomicLong();
         final PersistentMapStats previousStats = stats();
-        LOGGER.info("start reindexing file: {}, version: {}, stats before:\n{}", path, version, previousStats);
+        LOGGER.info("start reindexing file: {}, version: {}", path, version);
         final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp");
 
         try (PersistentMap<K, V> newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) {
@@ -595,7 +598,6 @@ public class PersistentMap<K, V> implements AutoCloseable {
             });
 
             final PersistentMapStats newStats = newMap.stats();
-            LOGGER.info("stats after reindex:\n{} ", newStats);
 
             if (previousStats.getValues() != newStats.getValues()) {
                 throw new IllegalStateException("reindex of " + path + " failed");
@@ -614,11 +616,12 @@ public class PersistentMap<K, V> implements AutoCloseable {
         final double durationInMs = (System.nanoTime() - start) / 1_000_000.0;
         final double valuesPerSecond = countValues.get() / (durationInMs / 1000);
         LOGGER.info("done reindexing, took {} ms, {} values, {} values/s, stats after:\n{}",
-                (int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats());
+                (int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats().diffView(previousStats));
     }
 
     public synchronized PersistentMapStats stats() {
         final PersistentMapStats stats = new PersistentMapStats();
+        stats.setFileSize(gatherFileSize());
 
         visitNodesPreOrder((node, depth) -> {
             stats.addDepth(depth);
@@ -640,6 +643,15 @@ public class PersistentMap<K, V> implements AutoCloseable {
         return stats;
     }
 
+    private long gatherFileSize() {
+        try {
+            return Files.size(path);
+        } catch (final IOException e) {
+            LOGGER.warn("failed to get file size for {}", path, e);
+            return -1;
+        }
+    }
+
     public synchronized boolean isEmpty() {
         final long rootNodeOffset = readNodeOffsetOfRootNode();
         final PersistentMapDiskNode node = getNode(rootNodeOffset);
@@ -691,6 +703,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
         try {
             if (version < 1) {
                 reindex();
+            } else {
+                final String reindexProperty = System.getProperty("pdb.reindex", "false");
+                if (!isNew && (reindexProperty.equals("true")
+                        || path.getParent().getFileName().toString().equals(reindexProperty))) {
+                    LOGGER.info("reindexing {} because system property 'pdb.reindex' was '{}'", path, reindexProperty);
+                    reindex();
+                }
             }
         } catch (final IOException e) {
             throw new IllegalStateException(

block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java

@@ -1,5 +1,7 @@
 package org.lucares.pdb.map;
 
+import java.util.List;
+
 public class PersistentMapStats {
     private long values = 0;
     private long nodes = 0;
@@ -15,11 +17,16 @@ public class PersistentMapStats {
 
     private double averageValuesInNode;
     private long innerNodes = 0;
+    private long fileSize;
 
     public PersistentMapStats() {
         super();
     }
 
+    public void setFileSize(final long size) {
+        this.fileSize = size;
+    }
+
     public long getValues() {
         return values;
     }
@@ -79,8 +86,35 @@ public class PersistentMapStats {
         builder.append("\nmaxDepth=   " + maxDepth);
         builder.append(String.format("\navg. depth= %.2f", averageDepth));
         builder.append(String.format("\navg. fill=  %.2f", averageFill));
-        builder.append(String.format("\nvalues/node=%.2f\n", averageValuesInNode));
+        builder.append(String.format("\nvalues/node=%.2f", averageValuesInNode));
+        builder.append(String.format("\nfile size=  %s\n", toHumanBytes(fileSize)));
         return builder.toString();
     }
 
+    public String diffView(final PersistentMapStats old) {
+        final StringBuilder builder = new StringBuilder();
+        builder.append("values=     " + old.values + " -> " + values);
+        builder.append("\nnodes=      " + old.nodes + " -> " + nodes);
+        builder.append("\ninnerNodes= " + old.innerNodes + " -> " + innerNodes);
+        builder.append("\nmaxDepth=   " + old.maxDepth + " -> " + maxDepth);
+        builder.append(String.format("\navg. depth= %.2f -> %.2f", old.averageDepth, averageDepth));
+        builder.append(String.format("\navg. fill=  %.2f -> %.2f", old.averageFill, averageFill));
+        builder.append(String.format("\nvalues/node=%.2f -> %.2f", old.averageValuesInNode, averageValuesInNode));
+        builder.append(String.format("\nfile size=  %s -> %s\n", toHumanBytes(old.fileSize), toHumanBytes(fileSize)));
+        return builder.toString();
+    }
+
+    private static String toHumanBytes(final long bytes) {
+        final List<String> powers = List.of("bytes", "KB", "MB", "GB", "TB", "PB", "EB");
+
+        int power = 1;
+        String result = String.format("%d bytes", bytes);
+        while (bytes >= Math.pow(1024, power) && power < powers.size()) {
+            result = String.format("%.3f", bytes / Math.pow(1024, power));
+            result = result.replaceAll("\\.?0*$", "");
+            result = result + " " + powers.get(power);
+            power = power + 1;
+        }
+        return result;
+    }
 }

data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java

@@ -6,7 +6,7 @@ import org.lucares.pdb.api.Tags;
 import org.lucares.pdb.datastore.Doc;
 import org.lucares.utils.byteencoder.VariableByteEncoder;
 
-class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
+public class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
 
     @Override
     public byte[] encode(final Doc doc) {
@@ -44,6 +44,7 @@ class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
         return t;
     }
 
+    @Override
     public byte[] getEmptyValue() {
         return new byte[] { 0 };
     }

data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java

@@ -13,6 +13,8 @@ import org.lucares.pdb.api.RuntimeIOException;
 import org.lucares.pdb.map.PersistentMap;
 import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
 import org.lucares.pdb.map.Visitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A wrapper for {@link PersistentMap} that partitions the values into several
@@ -25,6 +27,8 @@ import org.lucares.pdb.map.Visitor;
  */
 public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(PartitionPersistentMap.class);
+
     private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
 
     private final Function<ParititionId, PersistentMap<K, P>> creator;
@@ -47,7 +51,9 @@ public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
             }
             return null;
         };
+        final long start = System.nanoTime();
         preload(storageBasePath);
+        LOGGER.info("preloading {} took {}ms", filename, (System.nanoTime() - start) / 1_000_000.0);
     }
 
     private void preload(final Path storageBasePath) {