reduce memory usage

Reduce memory usage by storing the filename as string instead of
individual tags.

pdb-api/src/test/java/org/lucares/memory/MemoryScale.java

@@ -6,37 +6,57 @@ import java.nio.file.Paths;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-import org.lucares.utils.MiniMap;
+import org.lucares.pdb.api.StringCompressor;
+import org.lucares.pdb.api.Tag;
+import org.lucares.pdb.api.Tags;
+import org.lucares.pdb.api.UniqueStringIntegerPairs;
 
 public class MemoryScale {
-	
+
 	public static final String A = "A";
 
-	public static void main(String[] args) {
-		System.out.println("start");
+	public static void main(final String[] args) {
+		Tags.STRING_COMPRESSOR = new StringCompressor(new UniqueStringIntegerPairs());
+
+		scale("singleTag");
+		scale("tags0");
+		scale("tags1");
+		scale("tags2");
+		scale("tags6");
+	}
+
+	private static void scale(final String what) {
+		System.out.println("start: " + what);
 		// warmup of classes
 		getUsedMemory();
-		Object handle =createObject();
-		
-		handle = null;
-		
-		runGc();
-		long memoryBefore = getUsedMemory();
+		Object handle = createObject(what);
+
+		handle = null;
 
-		handle = createObject();
-		
 		runGc();
-		long memoryAfter = getUsedMemory();
-		System.out.println("used memory: " + (memoryAfter - memoryBefore));
+		final long memoryBefore = getUsedMemory();
+
+		handle = createObject(what);
+
+		runGc();
+		final long memoryAfter = getUsedMemory();
+		System.out.println(what + ": used memory: " + (memoryAfter - memoryBefore));
 		handle.hashCode(); // use the variable, so causes no warnings and is not removed by JIT compiler
 	}
-	
-	private static Object createObject(){
-		
-		String key = "pathAsUtf8";
-		switch (key) {
-		case "minimap":
-			return createMinimap();
+
+	private static Object createObject(final String what) {
+
+		switch (what) {
+		case "singleTag":
+			return createTag();
+		case "tags0":
+			return createTags0();
+		case "tags1":
+			return createTags1();
+		case "tags2":
+			return createTags2();
+		case "tags6":
+			return createTags6();
 		case "string":
 			return createString();
 		case "linkedHashMap":
@@ -50,45 +70,60 @@ public class MemoryScale {
 		default:
 			return null;
 		}
-		
+
 	}
 
-	private static Object createPathAsUtf8(String string) {
+	private static Object createTag() {
+		return new Tag("", "");
+	}
+
+	private static Object createTags0() {
+		return new Tags();
+	}
+
+	private static Object createTags1() {
+		return Tags.create("k1", "v1");
+	}
+
+	private static Object createTags2() {
+		return Tags.create("k1", "v1", "k2", "v2");
+	}
+
+	private static Object createTags6() {
+		Tags result = Tags.create("k1", "v1");
+		result = result.copyAdd("k2", "v2");
+		result = result.copyAdd("k3", "v3");
+		result = result.copyAdd("k4", "v4");
+		result = result.copyAdd("k5", "v5");
+		result = result.copyAdd("k6", "v6");
+		return result;
+	}
+
+	private static Object createPathAsUtf8(final String string) {
 		// TODO Auto-generated method stub
 		return string.getBytes(StandardCharsets.UTF_8);
 	}
 
-	private static String createPathAsString(String string) {
+	private static String createPathAsString(final String string) {
 		return string.replace("C", "c");
 	}
 
-	private static Path createPath(String string) {
+	private static Path createPath(final String string) {
 		return Paths.get(string);
 	}
 
-	private static Object createMinimap() {
-		final MiniMap<String, String> map = new MiniMap<>();
-
-		map.put("A", "A");
-		for (int i = 0; i < 0; i++){
-			map.put(""+i, ""+i);	
-		}
-		
-		return map;
-	}
-	
 	private static String createString() {
 
-		int i= 0;
-		return ""+i;
+		final int i = 0;
+		return "" + i;
 	}
 
 	private static Object createLinkedHashMap() {
 		final Map<String, String> map = new LinkedHashMap<>();
 
 		map.put("A", "A");
-		for (int i = 0; i < 0; i++){
-			map.put(""+i, ""+i);	
+		for (int i = 0; i < 0; i++) {
+			map.put("" + i, "" + i);
 		}
 
 		return map;
@@ -99,14 +134,13 @@ public class MemoryScale {
 			System.gc();
 			try {
 				Thread.sleep(100);
-			} catch (InterruptedException e) {
+			} catch (final InterruptedException e) {
 				Thread.currentThread().interrupt();
 			}
 		}
 	}
 
 	private static long getUsedMemory() {
-		return Runtime.getRuntime().totalMemory()
-				- Runtime.getRuntime().freeMemory();
+		return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
 	}
 }

pdb-api/src/test/java/org/lucares/pdb/api/RadixConverterTest.java

@@ -0,0 +1,24 @@
+package org.lucares.pdb.api;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+@Test
+public class RadixConverterTest {
+
+	public void testConvertRoundtrip() {
+
+		for (int i = 0; i < 1000; i++) {
+
+			final String string = RadixConverter.toString(i);
+			final int actual = RadixConverter.fromString(string);
+
+			Assert.assertEquals(actual, i, "string representation: " + string);
+		}
+	}
+
+	@Test(expectedExceptions = IllegalArgumentException.class)
+	public void testNoNegativeValues() {
+		RadixConverter.toString(-1);
+	}
+}

pdb-api/src/test/java/org/lucares/pdb/api/StringCompressorTest.java

@@ -0,0 +1,52 @@
+package org.lucares.pdb.api;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.lucares.utils.file.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test
+public class StringCompressorTest {
+	private Path dataDirectory;
+
+	@BeforeMethod
+	public void beforeMethod() throws IOException {
+		dataDirectory = Files.createTempDirectory("pdb");
+	}
+
+	@AfterMethod
+	public void afterMethod() throws IOException {
+		FileUtils.delete(dataDirectory);
+	}
+
+	public void testKeyCompressorRoundtrip() throws Exception {
+		final StringCompressor keyValueCompressor = StringCompressor.create(dataDirectory.resolve("key.csv"));
+
+		final String value = "foo";
+		final Integer intFoo = keyValueCompressor.put(value);
+		final String actual = keyValueCompressor.get(intFoo);
+
+		Assert.assertEquals(actual, value);
+	}
+
+	public void testKeyCompressorInitialization() throws Exception {
+		final Path database = dataDirectory.resolve("key.csv");
+		final String value = "foo";
+		{
+			final StringCompressor keyValueCompressor = StringCompressor.create(database);
+
+			keyValueCompressor.put(value);
+		}
+		{
+			final StringCompressor keyValueCompressor = StringCompressor.create(database);
+
+			keyValueCompressor.get(0);
+		}
+
+	}
+}

pdb-api/src/test/java/org/lucares/pdb/api/UniqueStringIntegerPairsTest.java

@@ -0,0 +1,74 @@
+package org.lucares.pdb.api;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.lucares.utils.file.FileUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test
+public class UniqueStringIntegerPairsTest {
+
+	private Path dataDirectory;
+
+	@BeforeMethod
+	public void beforeMethod() throws IOException {
+		dataDirectory = Files.createTempDirectory("pdb");
+	}
+
+	@AfterMethod
+	public void afterMethod() throws IOException {
+		FileUtils.delete(dataDirectory);
+	}
+
+	public void testPutGet() throws Exception {
+		final Path database = dataDirectory.resolve("key.csv");
+		final String first = "key1";
+		final Integer second = 1;
+
+		{
+			final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
+
+			usip.put(first, second);
+			Assert.assertEquals(usip.get(first), second);
+			Assert.assertEquals(usip.getKey(second), first);
+		}
+
+		{
+			final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
+
+			Assert.assertEquals(usip.get(first), second);
+			Assert.assertEquals(usip.getKey(second), first);
+		}
+	}
+
+	public void testUniqueKeyContstraint() throws Exception {
+		final Path database = dataDirectory.resolve("key.csv");
+		final String first = "key1";
+		final Integer second = 1;
+
+		final UniqueStringIntegerPairs usip = new UniqueStringIntegerPairs(database);
+		usip.put(first, second);
+		try {
+			// cannot add another pair with the first key
+			final int another = second + 1;
+			usip.put(first, another);
+			Assert.fail("expected an IllegalArgumentException");
+		} catch (final IllegalArgumentException e) {
+			// expected
+		}
+
+		try {
+			// cannot add another pair with the same second value
+			final String another = first + 1;
+			usip.put(another, second);
+			Assert.fail("expected an IllegalArgumentException");
+		} catch (final IllegalArgumentException e) {
+			// expected
+		}
+	}
+}