diff --git a/performanceDb/build.gradle b/performanceDb/build.gradle index 7121f16..b0a86d4 100644 --- a/performanceDb/build.gradle +++ b/performanceDb/build.gradle @@ -4,5 +4,6 @@ dependencies { compile 'org.lucares:ludb:1.0.20161002111352' //compile 'commons-io:commons-io:2.5' //compile 'joda-time:joda-time:2.9.6' + compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5' + compile 'org.mapdb:mapdb:3.0.2' } - diff --git a/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java b/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java new file mode 100644 index 0000000..e4d6b13 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/ObjectMapperTest.java @@ -0,0 +1,320 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.io.Serializable; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.lucares.ludb.FieldType; +import org.lucares.ludb.H2DB; +import org.mapdb.DB; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; +import org.testng.annotations.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; + +import liquibase.exception.LiquibaseException; + +@Test +public class ObjectMapperTest { + private static final File H2DB_LOCATION = new File("/tmp/h2.db"); + + public static class TagEntry implements Serializable { + private static final long serialVersionUID = 1L; + + private String folder; + + private Map tags = new HashMap<>(); + + public TagEntry() { + super(); + } + + public Path getFolder() { + return Paths.get(folder); + } + + public void setFolder(final Path folder) { + this.folder = folder.toFile().getPath(); + } + + public Map getTags() { + return tags; + } + + public void setTags(final Map tags) { + this.tags = tags; + } + + public void tag(final String key, final String value) { + tags.put(key, value); + } + } + + public static interface TagDB { + + void addTagEntry(TagEntry tagEntry); + + } + + public static class TagDBJson implements TagDB { + private List tagEntries = new ArrayList<>(); + + private final Map>> index = new HashMap<>(); + + public Map>> index() { + return index; + } + + @Override + public void addTagEntry(final TagEntry tagEntry) { + tagEntries.add(tagEntry); + } + + public List getTagEntries() { + return tagEntries; + } + + public void setTagEntries(final List tagEntries) { + this.tagEntries = tagEntries; + + index.clear(); + + for (final TagEntry tagEntry : tagEntries) { + for (final Entry t : tagEntry.getTags().entrySet()) { + + final String tagKey = t.getKey(); + final String tagValue = t.getValue(); + index.putIfAbsent(tagKey, new HashMap<>()); + + final Map> tagKeyMap = index.get(tagKey); + tagKeyMap.putIfAbsent(tagValue, new HashSet<>()); + + tagKeyMap.get(tagValue).add(tagEntry); + } + } + } + } + + public static class TagDBH2 implements TagDB, AutoCloseable { + + private final H2DB db; + + public TagDBH2() throws LiquibaseException { + super(); + this.db = new H2DB(H2DB_LOCATION); + } + + @Override + public void addTagEntry(final TagEntry tagEntry) { + final File file = tagEntry.getFolder().toFile(); + db.addDocument(file); + + tagEntry.getTags().forEach((k, v) -> { + try { + db.setProperty(file, k, v); + } catch (final NullPointerException e) { + db.createField(k, FieldType.STRING); + db.setProperty(file, k, v); + } + }); + + } + + public List getTagEntries() { + return null; + } + + public void setTagEntries(final List tagEntries) { + tagEntries.forEach(this::addTagEntry); + } + + @Override + public void close() throws Exception { + db.close(); + } + } + + public static class TagDBMapDB implements TagDB, AutoCloseable { + private List tagEntries = new ArrayList<>(); + + final DB db; + + private final ConcurrentMap mapIndex; + + private final ConcurrentMap mapObjects; + + public TagDBMapDB() { + db = DBMaker.fileDB("/tmp/file4.db").fileMmapEnable().make(); + + mapIndex = db.hashMap("mapIndex", Serializer.STRING, Serializer.LONG).createOrOpen(); + + mapObjects = db.hashMap("mapObjects", Serializer.LONG, Serializer.JAVA).createOrOpen(); + } + + @Override + public void close() throws Exception { + db.close(); + } + + @Override + public void addTagEntry(final TagEntry tagEntry) { + + final long docId = mapObjects.size(); + mapObjects.put(docId, tagEntry); + + tagEntry.getTags().forEach((k, v) -> { + final String tag = k + "=" + v; + + mapIndex.put(tag, docId); + + }); + + } + + public List getTagEntries() { + return tagEntries; + } + + public void setTagEntries(final List tagEntries) { + this.tagEntries = tagEntries; + + for (final TagEntry tagEntry : tagEntries) { + for (final Entry t : tagEntry.getTags().entrySet()) { + + final String tagKey = t.getKey(); + final String tagValue = t.getValue(); + final String tag = tagKey + "=" + tagValue; + + final long docId = mapObjects.size(); + mapObjects.put(docId, tagEntry); + + mapIndex.put(tag, docId); + } + } + } + } + + @Test(invocationCount = 1) + public void testName() throws Exception { + + final TagDBJson tagDB = new TagDBJson(); + fill(tagDB); + + final ObjectMapper om = new ObjectMapper(); + final ObjectWriter ow = om.writerWithDefaultPrettyPrinter(); + + final File file = new File("/tmp/example.json"); + { + final long start = System.nanoTime(); + ow.writeValue(file, tagDB); + System.out.println("json write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + } + { + final long start = System.nanoTime(); + final TagDBJson actualTagDB = om.readValue(file, TagDBJson.class); + System.out.println("json read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + + } + } + + @Test(invocationCount = 1) + public void testH2BD() throws Exception { + + H2DB_LOCATION.delete(); + { + final long start = System.nanoTime(); + final TagDBH2 tagDB = new TagDBH2(); + fill(tagDB); + System.out.println("h2 write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + } + { + final long start = System.nanoTime(); + final TagDBH2 tagDB = new TagDBH2(); + System.out.println("h2 read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + + } + } + + @Test(enabled = true) + public void testMapDB() throws Exception { + + final long start = System.nanoTime(); + try (final TagDBMapDB tagDB = new TagDBMapDB()) { + + fill(tagDB); + + } + System.out.println("mapdb write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms"); + + final long start2 = System.nanoTime(); + try (final TagDBMapDB tagDB = new TagDBMapDB()) { + + } + System.out.println("mapdb read: " + (System.nanoTime() - start2) / 1_000_000.0 + "ms"); + } + + @Test(enabled = false) + public void testQuickstart() throws Exception { + final DB db = DBMaker.fileDB("/tmp/file2.db").fileMmapEnable().make(); + final ConcurrentMap map = db.hashMap("map", Serializer.STRING, Serializer.LONG).createOrOpen(); + map.put("something", 111L); + + db.close(); + } + + private void fill(final TagDB tagDB) { + + final List pods = IntStream.rangeClosed(1, 10).mapToObj(i -> "vapondem" + i) + .collect(Collectors.toList()); + final List hosts = IntStream.rangeClosed(1, 10).mapToObj(i -> "host" + i).collect(Collectors.toList()); + final List versions = IntStream.rangeClosed(1, 10).mapToObj(i -> "5." + i).collect(Collectors.toList()); + final List types = Arrays.asList("app", "engine", "web", "batch"); + + for (final String pod : pods) { + for (final String host : hosts) { + for (final String version : versions) { + for (final String type : types) { + final TagEntry e = new TagEntry(); + + e.setFolder(Paths.get("/some/prefix", UUID.randomUUID().toString())); + e.tag("pod", pod); + e.tag("host", host); + e.tag("version", version); + e.tag("type", type); + + tagDB.addTagEntry(e); + } + } + + } + } + } + + public static void main(final String[] args) throws Exception { + // System.out.println(System.getProperty("java.class.path")); + TimeUnit.SECONDS.sleep(5); + try (H2DB h2 = new H2DB(new File("/tmp/h2" + UUID.randomUUID().toString()))) { + System.out.println("done"); + } + + TimeUnit.SECONDS.sleep(10); + System.out.println("closing"); + } + +}