andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

make it possible to reindex the persistent maps via system property

Browse Source
This commit is contained in:
Andreas Huber
2023-02-11 18:34:21 +01:00
parent 728fe1df78
commit bd8ab49b71
4 changed files with 65 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java
View File

@@ -188,6 +188,8 @@ public class PersistentMap<K, V> implements AutoCloseable {
private long version;
private boolean isNew;
/**
*
* @param path file for the index, must be child of storageBasePath
@@ -218,6 +220,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
private void initIfNew() {
if (diskStore.size() < BLOCK_SIZE) {
isNew = true;
final long nodeOffsetToRootNode = diskStore.allocateBlock(diskStore.minAllocationSize());
Preconditions.checkEqual(nodeOffsetToRootNode, OFFSET_META_DATA,
"The offset of the pointer to the root node must be at a well known location. "
@@ -582,7 +585,7 @@ public class PersistentMap<K, V> implements AutoCloseable {
final long start = System.nanoTime();
final AtomicLong countValues = new AtomicLong();
final PersistentMapStats previousStats = stats();
LOGGER.info("start reindexing file: {}, version: {}, stats before:\n{}", path, version, previousStats);
LOGGER.info("start reindexing file: {}, version: {}", path, version);
final Path newFile = path.getParent().resolve(path.getFileName() + ".tmp");
try (PersistentMap<K, V> newMap = new PersistentMap<>(newFile, null, keyEncoder, valueEncoder)) {
@@ -595,7 +598,6 @@ public class PersistentMap<K, V> implements AutoCloseable {
});
final PersistentMapStats newStats = newMap.stats();
LOGGER.info("stats after reindex:\n{} ", newStats);
if (previousStats.getValues() != newStats.getValues()) {
throw new IllegalStateException("reindex of " + path + " failed");
@@ -614,11 +616,12 @@ public class PersistentMap<K, V> implements AutoCloseable {
final double durationInMs = (System.nanoTime() - start) / 1_000_000.0;
final double valuesPerSecond = countValues.get() / (durationInMs / 1000);
LOGGER.info("done reindexing, took {} ms, {} values, {} values/s, stats after:\n{}",
(int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats());
(int) Math.ceil(durationInMs), countValues.get(), valuesPerSecond, stats().diffView(previousStats));
}
public synchronized PersistentMapStats stats() {
final PersistentMapStats stats = new PersistentMapStats();
stats.setFileSize(gatherFileSize());
visitNodesPreOrder((node, depth) -> {
stats.addDepth(depth);
@@ -640,6 +643,15 @@ public class PersistentMap<K, V> implements AutoCloseable {
return stats;
}
private long gatherFileSize() {
try {
return Files.size(path);
} catch (final IOException e) {
LOGGER.warn("failed to get file size for {}", path, e);
return -1;
}
}
public synchronized boolean isEmpty() {
final long rootNodeOffset = readNodeOffsetOfRootNode();
final PersistentMapDiskNode node = getNode(rootNodeOffset);
@@ -691,6 +703,13 @@ public class PersistentMap<K, V> implements AutoCloseable {
try {
if (version < 1) {
reindex();
} else {
final String reindexProperty = System.getProperty("pdb.reindex", "false");
if (!isNew && (reindexProperty.equals("true")
|| path.getParent().getFileName().toString().equals(reindexProperty))) {
LOGGER.info("reindexing {} because system property 'pdb.reindex' was '{}'", path, reindexProperty);
reindex();
}
}
} catch (final IOException e) {
throw new IllegalStateException(

36
block-storage/src/main/java/org/lucares/pdb/map/PersistentMapStats.java
View File

@@ -1,5 +1,7 @@
package org.lucares.pdb.map;
import java.util.List;
public class PersistentMapStats {
private long values = 0;
private long nodes = 0;
@@ -15,11 +17,16 @@ public class PersistentMapStats {
private double averageValuesInNode;
private long innerNodes = 0;
private long fileSize;
public PersistentMapStats() {
super();
}
public void setFileSize(final long size) {
this.fileSize = size;
}
public long getValues() {
return values;
}
@@ -79,8 +86,35 @@ public class PersistentMapStats {
builder.append("\nmaxDepth= " + maxDepth);
builder.append(String.format("\navg. depth= %.2f", averageDepth));
builder.append(String.format("\navg. fill= %.2f", averageFill));
builder.append(String.format("\nvalues/node=%.2f\n", averageValuesInNode));
builder.append(String.format("\nvalues/node=%.2f", averageValuesInNode));
builder.append(String.format("\nfile size= %s\n", toHumanBytes(fileSize)));
return builder.toString();
}
public String diffView(final PersistentMapStats old) {
final StringBuilder builder = new StringBuilder();
builder.append("values= " + old.values + " -> " + values);
builder.append("\nnodes= " + old.nodes + " -> " + nodes);
builder.append("\ninnerNodes= " + old.innerNodes + " -> " + innerNodes);
builder.append("\nmaxDepth= " + old.maxDepth + " -> " + maxDepth);
builder.append(String.format("\navg. depth= %.2f -> %.2f", old.averageDepth, averageDepth));
builder.append(String.format("\navg. fill= %.2f -> %.2f", old.averageFill, averageFill));
builder.append(String.format("\nvalues/node=%.2f -> %.2f", old.averageValuesInNode, averageValuesInNode));
builder.append(String.format("\nfile size= %s -> %s\n", toHumanBytes(old.fileSize), toHumanBytes(fileSize)));
return builder.toString();
}
private static String toHumanBytes(final long bytes) {
final List<String> powers = List.of("bytes", "KB", "MB", "GB", "TB", "PB", "EB");
int power = 1;
String result = String.format("%d bytes", bytes);
while (bytes >= Math.pow(1024, power) && power < powers.size()) {
result = String.format("%.3f", bytes / Math.pow(1024, power));
result = result.replaceAll("\\.?0*$", "");
result = result + " " + powers.get(power);
power = power + 1;
}
return result;
}
}

3
data-store/src/main/java/org/lucares/pdb/datastore/internal/DocEncoderDecoder.java
View File

@@ -6,7 +6,7 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Doc;
import org.lucares.utils.byteencoder.VariableByteEncoder;
class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
public class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
@Override
public byte[] encode(final Doc doc) {
@@ -44,6 +44,7 @@ class DocEncoderDecoder implements PartitionAwareEncoderDecoder<Doc, Doc> {
return t;
}
@Override
public byte[] getEmptyValue() {
return new byte[] { 0 };
}

6
data-store/src/main/java/org/lucares/pdb/datastore/internal/PartitionPersistentMap.java
View File

@@ -13,6 +13,8 @@ import org.lucares.pdb.api.RuntimeIOException;
import org.lucares.pdb.map.PersistentMap;
import org.lucares.pdb.map.PersistentMap.EncoderDecoder;
import org.lucares.pdb.map.Visitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* A wrapper for {@link PersistentMap} that partitions the values into several
@@ -25,6 +27,8 @@ import org.lucares.pdb.map.Visitor;
*/
public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(PartitionPersistentMap.class);
private final ConcurrentHashMap<ParititionId, PersistentMap<K, P>> maps = new ConcurrentHashMap<>();
private final Function<ParititionId, PersistentMap<K, P>> creator;
@@ -47,7 +51,9 @@ public class PartitionPersistentMap<K, V, P> implements AutoCloseable {
}
return null;
};
final long start = System.nanoTime();
preload(storageBasePath);
LOGGER.info("preloading {} took {}ms", filename, (System.nanoTime() - start) / 1_000_000.0);
}
private void preload(final Path storageBasePath) {