diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java index 451dd70..ee2b152 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java @@ -20,6 +20,7 @@ import java.util.function.Function; import org.lucares.collections.LongList; import org.lucares.pdb.diskstorage.DiskBlock; import org.lucares.pdb.diskstorage.DiskStorage; +import org.lucares.utils.ArrayUtils; import org.lucares.utils.Preconditions; import org.lucares.utils.byteencoder.VariableByteEncoder; import org.lucares.utils.cache.LRUCache; @@ -41,6 +42,10 @@ public class PersistentMap implements AutoCloseable { void visit(PersistentMapDiskNode node, PersistentMapDiskNode parentNode, NodeEntry nodeEntry, int depth); } + interface NodeVisitorCallback { + void visit(PersistentMapDiskNode node, int depth); + } + public interface EncoderDecoder { public byte[] encode(O object); @@ -453,7 +458,9 @@ public class PersistentMap implements AutoCloseable { for (final NodeEntry child : node.getEntries()) { - visitor.visit(node, parentNode, child, depth); + if (Arrays.compare(child.getKey(), MAX_KEY) != 0) { + visitor.visit(node, parentNode, child, depth); + } if (child.isInnerNode()) { final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue()); visitNodeEntriesPreOrderRecursively(childNodeOffset, node, visitor, depth + 1); @@ -461,10 +468,35 @@ public class PersistentMap implements AutoCloseable { } } + private synchronized void visitNodesPreOrder(final NodeVisitorCallback visitor) { + final long rootNodeOffset = readNodeOffsetOfRootNode(); + visitNodesPreOrderRecursively(rootNodeOffset, visitor, 0); + } + + private void visitNodesPreOrderRecursively(final long nodeOffset, final NodeVisitorCallback visitor, + final int depth) { + final PersistentMapDiskNode node = getNode(nodeOffset); + visitor.visit(node, depth); + + for (final NodeEntry child : node.getEntries()) { + if (child.isInnerNode()) { + final long childNodeOffset = VariableByteEncoder.decodeFirstValue(child.getValue()); + visitNodesPreOrderRecursively(childNodeOffset, visitor, depth + 1); + } + } + } + enum VisitByPrefixMode { FIND, ITERATE } + public synchronized void forAll(final Visitor visitor) { + final byte[] encodedKeyPrefix = new byte[0]; + + final long rootNodeOffset = readNodeOffsetOfRootNode(); + iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, visitor); + } + public synchronized void visitValues(final K keyPrefix, final Visitor visitor) { final byte[] encodedKeyPrefix = keyEncoder.encode(keyPrefix); @@ -479,9 +511,7 @@ public class PersistentMap implements AutoCloseable { final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp"); try (PersistentMap newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) { - final long rootNodeOffset = readNodeOffsetOfRootNode(); - final byte[] encodedKeyPrefix = new byte[0]; - iterateNodeEntryByPrefix(rootNodeOffset, encodedKeyPrefix, (k, v) -> { + forAll((k, v) -> { newMap.putValue(k, v); final long count = countValues.incrementAndGet(); if (count % 100000 == 0) { @@ -494,12 +524,9 @@ public class PersistentMap implements AutoCloseable { valueCache.clear(); nodeCache.clear(); - final Path backupFile = path.getParent().resolve(path.getFileName() + "." - + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").format(OffsetDateTime.now()) + ".backup"); - Files.move(path, backupFile); - Files.move(newFile, path); + swapFiles(newFile); - this.diskStore = new DiskStorage(path, null); + diskStore = new DiskStorage(path, null); readOffsetOfRootNode(); final double durationInMs = (System.nanoTime() - start) / 1_000_000.0; final double valuesPerSecond = countValues.get() / (durationInMs / 1000); @@ -507,6 +534,36 @@ public class PersistentMap implements AutoCloseable { countValues.get(), valuesPerSecond); } + public synchronized PersistentMapStats stats() { + final PersistentMapStats stats = new PersistentMapStats(); + + visitNodesPreOrder((node, depth) -> { + stats.addDepth(depth); + stats.incrementNodes(node.isInnerNode()); + stats.addValuesInNode(node.getEntries().size()); + + final byte[] serialized = node.serialize(); + final int usedBytes = BLOCK_SIZE + - ArrayUtils.countConsecutive(serialized, (byte) 0, ArrayUtils.findFirst(serialized, (byte) 0)); + stats.addFill(usedBytes / (double) BLOCK_SIZE); + + for (final NodeEntry entry : node.getEntries()) { + if (entry.isDataNode()) { + stats.incrementValues(); + } + } + }); + + return stats; + } + + private void swapFiles(final Path newFile) throws IOException { + final Path backupFile = path.getParent().resolve(path.getFileName() + "." + + DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").format(OffsetDateTime.now()) + ".backup"); + Files.move(path, backupFile); + Files.move(newFile, path); + } + private void iterateNodeEntryByPrefix(final long nodeOffest, final byte[] keyPrefix, final Visitor visitor) { final PersistentMapDiskNode node = getNode(nodeOffest); @@ -525,8 +582,6 @@ public class PersistentMap implements AutoCloseable { final K key = keyEncoder.decode(entry.getKey()); final V value = valueEncoder.decode(entry.getValue()); visitor.visit(key, value); - - // System.out.println("--> " + key + "=" + value); } else if (prefixCompareResult > 0) { break; } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java index 133e37a..25117f1 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapDiskNode.java @@ -294,4 +294,9 @@ public class PersistentMapDiskNode { } } + public boolean isInnerNode() { + + return entries.size() > 0 ? entries.firstEntry().getValue().isInnerNode() : false; + } + } diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java new file mode 100644 index 0000000..3cafc11 --- /dev/null +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java @@ -0,0 +1,66 @@ +package org.lucares.pdb.map; + +public class PersistentMapStats { + private long values = 0; + private long nodes = 0; + + private long maxDepth = 0; + private long countDepth = 0; + private double averageDepth = 0; + + private long countFill = 0; + private double averageFill = 0; + + private int countValuesInNode; + + private double averageValuesInNode; + private long innerNodes = 0; + + public PersistentMapStats() { + super(); + } + + public void incrementValues() { + values++; + } + + public void incrementNodes(final boolean innerNode) { + nodes++; + if (innerNode) { + innerNodes++; + } + } + + public void addDepth(final long depth) { + maxDepth = Math.max(maxDepth, depth); + + countDepth++; + averageDepth = averageDepth * (countDepth - 1) / countDepth + depth / (double) countDepth; + } + + public void addFill(final double fill) { + // online algorithm to compute average values: + countFill++; + averageFill = averageFill * (countFill - 1) / countFill + fill / countFill; + } + + public void addValuesInNode(final int valuesInNode) { + countValuesInNode++; + averageValuesInNode = averageValuesInNode * (countValuesInNode - 1) / countValuesInNode + + valuesInNode / (double) countValuesInNode; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("values= " + values); + builder.append("\nnodes= " + nodes); + builder.append("\ninnerNodes= " + innerNodes); + builder.append("\nmaxDepth= " + maxDepth); + builder.append(String.format("\navg. depth= %.2f", averageDepth)); + builder.append(String.format("\navg. fill= %.2f", averageFill)); + builder.append(String.format("\nvalues/node=%.2f\n", averageValuesInNode)); + return builder.toString(); + } + +} diff --git a/pdb-utils/src/main/java/org/lucares/utils/ArrayUtils.java b/pdb-utils/src/main/java/org/lucares/utils/ArrayUtils.java new file mode 100644 index 0000000..cd05355 --- /dev/null +++ b/pdb-utils/src/main/java/org/lucares/utils/ArrayUtils.java @@ -0,0 +1,24 @@ +package org.lucares.utils; + +public class ArrayUtils { + public static int findFirst(final byte[] bytes, final byte key) { + for (int i = 0; i < bytes.length; i++) { + if (bytes[i] == key) { + return i; + } + } + return -1; + } + + public static int countConsecutive(final byte[] bytes, final byte key, final int offset) { + int count = 0; + for (int i = offset; i < bytes.length; i++) { + if (bytes[i] == key) { + count++; + } else { + break; + } + } + return count; + } +}