diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java index 584842d..655f000 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java @@ -188,6 +188,8 @@ public class PersistentMap implements AutoCloseable { private long version; + private boolean isNew; + /** * * @param path file for the index, must be child of storageBasePath @@ -218,6 +220,7 @@ public class PersistentMap implements AutoCloseable { private void initIfNew() { if (diskStore.size() < BLOCK_SIZE) { + isNew = true; final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize()); Preconditions.checkEqual(nodeOffsetToRootNode, OFFSET_META_DATA, "The offset of the pointer to the root node must be at a well known location. " @@ -582,7 +585,7 @@ public class PersistentMap implements AutoCloseable { final long start = System.nanoTime(); final AtomicLong countValues = new AtomicLong(); final PersistentMapStats previousStats = stats(); - LOGGER.info("start reindexing file: {}, version: {}, stats before:\n{}", path, version, previousStats); + LOGGER.info("start reindexing file: {}, version: {}", path, version); final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp"); try (PersistentMap newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) { @@ -595,7 +598,6 @@ public class PersistentMap implements AutoCloseable { }); final PersistentMapStats newStats = newMap.stats(); - LOGGER.info("stats after reindex:\n{} ", newStats); if (previousStats.getValues() != newStats.getValues()) { throw new IllegalStateException("reindex of " + path + " failed"); @@ -614,11 +616,12 @@ public class PersistentMap implements AutoCloseable { final double durationInMs = (System.nanoTime() - start) / 1_000_000.0; final double valuesPerSecond = countValues.get() / (durationInMs / 1000); LOGGER.info("done reindexing, took {} ms, {} values, {} values/s, stats after:\n{}", - (int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats()); + (int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats().diffView(previousStats)); } public synchronized PersistentMapStats stats() { final PersistentMapStats stats = new PersistentMapStats(); + stats.setFileSize(gatherFileSize()); visitNodesPreOrder((node, depth) -> { stats.addDepth(depth); @@ -640,6 +643,15 @@ public class PersistentMap implements AutoCloseable { return stats; } + private long gatherFileSize() { + try { + return Files.size(path); + } catch (final IOException e) { + LOGGER.warn("failed to get file size for {}", path, e); + return -1; + } + } + public synchronized boolean isEmpty() { final long rootNodeOffset = readNodeOffsetOfRootNode(); final PersistentMapDiskNode node = getNode(rootNodeOffset); @@ -691,6 +703,13 @@ public class PersistentMap implements AutoCloseable { try { if (version < 1) { reindex(); + } else { + final String reindexProperty = System.getProperty("pdb.reindex", "false"); + if (!isNew && (reindexProperty.equals("true") + || path.getParent().getFileName().toString().equals(reindexProperty))) { + LOGGER.info("reindexing {} because system property 'pdb.reindex' was '{}'", path, reindexProperty); + reindex(); + } } } catch (final IOException e) { throw new IllegalStateException( diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java index f8c7c7f..d53811b 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java @@ -1,5 +1,7 @@ package org.lucares.pdb.map; +import java.util.List; + public class PersistentMapStats { private long values = 0; private long nodes = 0; @@ -15,11 +17,16 @@ public class PersistentMapStats { private double averageValuesInNode; private long innerNodes = 0; + private long fileSize; public PersistentMapStats() { super(); } + public void setFileSize(final long size) { + this.fileSize = size; + } + public long getValues() { return values; } @@ -79,8 +86,35 @@ public class PersistentMapStats { builder.append("\nmaxDepth= " + maxDepth); builder.append(String.format("\navg. depth= %.2f", averageDepth)); builder.append(String.format("\navg. fill= %.2f", averageFill)); - builder.append(String.format("\nvalues/node=%.2f\n", averageValuesInNode)); + builder.append(String.format("\nvalues/node=%.2f", averageValuesInNode)); + builder.append(String.format("\nfile size= %s\n", toHumanBytes(fileSize))); return builder.toString(); } + public String diffView(final PersistentMapStats old) { + final StringBuilder builder = new StringBuilder(); + builder.append("values= " + old.values + " -> " + values); + builder.append("\nnodes= " + old.nodes + " -> " + nodes); + builder.append("\ninnerNodes= " + old.innerNodes + " -> " + innerNodes); + builder.append("\nmaxDepth= " + old.maxDepth + " -> " + maxDepth); + builder.append(String.format("\navg. depth= %.2f -> %.2f", old.averageDepth, averageDepth)); + builder.append(String.format("\navg. fill= %.2f -> %.2f", old.averageFill, averageFill)); + builder.append(String.format("\nvalues/node=%.2f -> %.2f", old.averageValuesInNode, averageValuesInNode)); + builder.append(String.format("\nfile size= %s -> %s\n", toHumanBytes(old.fileSize), toHumanBytes(fileSize))); + return builder.toString(); + } + + private static String toHumanBytes(final long bytes) { + final List powers = List.of("bytes", "KB", "MB", "GB", "TB", "PB", "EB"); + + int power = 1; + String result = String.format("%d bytes", bytes); + while (bytes >= Math.pow(1024, power) && power < powers.size()) { + result = String.format("%.3f", bytes / Math.pow(1024, power)); + result = result.replaceAll("\\.?0*$", ""); + result = result + " " + powers.get(power); + power = power + 1; + } + return result; + } } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java index 1bfaf62..a5c1093 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java @@ -6,7 +6,7 @@ import org.lucares.pdb.api.Tags; import org.lucares.pdb.datastore.Doc; import org.lucares.utils.byteencoder.VariableByteEncoder; -class DocEncoderDecoder implements PartitionAwareEncoderDecoder { +public class DocEncoderDecoder implements PartitionAwareEncoderDecoder { @Override public byte[] encode(final Doc doc) { @@ -44,6 +44,7 @@ class DocEncoderDecoder implements PartitionAwareEncoderDecoder { return t; } + @Override public byte[] getEmptyValue() { return new byte[] { 0 }; } diff --git a/data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java b/data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java index 899dfc1..aad3c35 100644 --- a/data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java +++ b/data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java @@ -13,6 +13,8 @@ import org.lucares.pdb.api.RuntimeIOException; import org.lucares.pdb.map.PersistentMap; import org.lucares.pdb.map.PersistentMap.EncoderDecoder; import org.lucares.pdb.map.Visitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A wrapper for {@link PersistentMap} that partitions the values into several @@ -25,6 +27,8 @@ import org.lucares.pdb.map.Visitor; */ public class PartitionPersistentMap implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(PartitionPersistentMap.class); + private final ConcurrentHashMap> maps = new ConcurrentHashMap<>(); private final Function> creator; @@ -47,7 +51,9 @@ public class PartitionPersistentMap implements AutoCloseable { } return null; }; + final long start = System.nanoTime(); preload(storageBasePath); + LOGGER.info("preloading {} took {}ms", filename, (System.nanoTime() - start) / 1_000_000.0); } private void preload(final Path storageBasePath) {