diff --git a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java index 66c3254..584842d 100644 --- a/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java +++ b/block-storage/src/main/java/org/lucares/pdb/map/PersistentMap.java @@ -190,7 +190,7 @@ public class PersistentMap implements AutoCloseable { /** * - * @param path file relative to {@code storageBasePath} + * @param path file for the index, must be child of storageBasePath * @param storageBasePath base path * @param keyEncoder encoder for keys * @param valueEncoder encoder for values diff --git a/pdb-api/build.gradle b/pdb-api/build.gradle index f7960ff..c891eb5 100644 --- a/pdb-api/build.gradle +++ b/pdb-api/build.gradle @@ -1,5 +1,6 @@ dependencies { implementation project(':pdb-utils') + implementation project(':block-storage') implementation lib_primitive_collections } \ No newline at end of file diff --git a/pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java b/pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java index 61a8cf5..007a514 100644 --- a/pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java +++ b/pdb-api/src/main/java/org/lucares/pdb/api/UniqueStringIntegerPairs.java @@ -2,11 +2,8 @@ package org.lucares.pdb.api; import java.io.BufferedReader; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.Writer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -19,6 +16,8 @@ import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; +import org.lucares.pdb.map.PersistentMap; + /** * A very simple {@link Set}-like or {@link Map}-like data structure that stores * unique¹ pairs of Strings and integers persistently. @@ -33,8 +32,6 @@ import java.util.regex.Pattern; public class UniqueStringIntegerPairs { private static final String SEPARATOR = "\t"; - private static final boolean APPEND = true; - private static final class ByteArray implements Comparable { private final byte[] array; private final int start; @@ -97,37 +94,50 @@ public class UniqueStringIntegerPairs { */ private final List intToString = new ArrayList<>(); - private final Path file; + final PersistentMap persistentMap; public UniqueStringIntegerPairs() { this(null); } public UniqueStringIntegerPairs(final Path storageBasePath) { - this.file = keyCompressionFile(storageBasePath); - if (file != null) { - init(file); + + if (storageBasePath != null) { + persistentMap = new PersistentMap<>(storageBasePath.resolve("keys.bs"), storageBasePath, + PersistentMap.STRING_CODER, PersistentMap.LONG_CODER); + + final Path oldKeysCsvFile = keyCompressionFile(storageBasePath); + if (persistentMap.isEmpty() && Files.exists(oldKeysCsvFile)) { + upgradeFromCsvFile(oldKeysCsvFile); + } else { + init(); + } + } else { + // some unit tests disable the persistence and use this class memory only + persistentMap = null; } } + private void init() { + persistentMap.forAll((string, integer) -> { + intToStringPut(integer.intValue(), string); + stringToInt.put(string, integer.intValue()); + bytesToInt.put(new ByteArray(string), integer.intValue()); + }); + } + private Path keyCompressionFile(final Path dataDirectory) { return dataDirectory.resolve("keys.csv"); } - private void init(final Path file) throws RuntimeIOException { + private void upgradeFromCsvFile(final Path file) throws RuntimeIOException { try { - Files.createDirectories(file.getParent()); - if (!Files.exists(file)) { - Files.createFile(file); - } - try (final BufferedReader reader = new BufferedReader( new InputStreamReader(new FileInputStream(file.toFile()), StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { - // TODO use more efficient code to read the CSV -> improves startup time final String[] tokens = line.split(Pattern.quote(SEPARATOR)); if (tokens.length == 2) { @@ -136,6 +146,7 @@ public class UniqueStringIntegerPairs { intToStringPut(integer, string); stringToInt.put(string, integer); bytesToInt.put(new ByteArray(string), integer); + persistentMap.putValue(string, (long) integer); } } } @@ -159,15 +170,8 @@ public class UniqueStringIntegerPairs { if (stringToInt.containsKey(string) || (intToString.size() > integer && intToString.get(integer) != null)) { throw new IllegalArgumentException("Unique key constraint violation for (" + string + ", " + integer + ")"); } - if (file != null) { - try (final Writer writer = new OutputStreamWriter(new FileOutputStream(file.toFile(), APPEND), - StandardCharsets.UTF_8)) { - - writer.write(string + SEPARATOR + integer + "\n"); - - } catch (final IOException e) { - throw new RuntimeIOException(e); - } + if (persistentMap != null) { + persistentMap.putValue(string, (long) integer); } intToStringPut(integer, string);