andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

move creation of PdbWriter to the DataStore

Browse Source
This commit is contained in:
Andreas Huber
2019-02-07 18:06:41 +01:00
parent 58bfba23bb
commit ea5884a5e6
14 changed files with 87 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
performanceDb/src/main/java/org/lucares/performance/db/InvalidValueException.java → data-store/src/main/java/org/lucares/pdb/datastore/InvalidValueException.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
public class InvalidValueException extends IllegalArgumentException {

4
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java → data-store/src/main/java/org/lucares/pdb/datastore/PdbFile.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
import java.io.IOException;
import java.util.List;
@@ -11,7 +11,7 @@ import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.diskstorage.DiskStorage;
class PdbFile {
public class PdbFile {
private static class PdbFileToLongStream implements Function<PdbFile, Stream<LongList>> {

6
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java → data-store/src/main/java/org/lucares/pdb/datastore/PdbWriter.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
import java.io.Flushable;
import java.io.IOException;
@@ -13,7 +13,7 @@ import org.slf4j.LoggerFactory;
/**
*
*/
class PdbWriter implements AutoCloseable, Flushable {
public class PdbWriter implements AutoCloseable, Flushable {
private static final Logger LOGGER = LoggerFactory.getLogger(PdbWriter.class);
@@ -22,7 +22,7 @@ class PdbWriter implements AutoCloseable, Flushable {
private final BSFile bsFile;
PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) throws IOException {
public PdbWriter(final PdbFile pdbFile, final DiskStorage diskStorage) throws IOException {
this.pdbFile = pdbFile;
bsFile = BSFile.existingFile(pdbFile.getRootBlockNumber(), diskStorage);

2
performanceDb/src/main/java/org/lucares/performance/db/ReadException.java → data-store/src/main/java/org/lucares/pdb/datastore/ReadException.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
import java.io.IOException;

2
performanceDb/src/main/java/org/lucares/performance/db/ReadRuntimeException.java → data-store/src/main/java/org/lucares/pdb/datastore/ReadRuntimeException.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
public class ReadRuntimeException extends RuntimeException {

2
performanceDb/src/main/java/org/lucares/performance/db/WriteException.java → data-store/src/main/java/org/lucares/pdb/datastore/WriteException.java
View File

@@ -1,4 +1,4 @@
package org.lucares.performance.db;
package org.lucares.pdb.datastore;
public class WriteException extends RuntimeException {

54
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -9,6 +9,7 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -21,7 +22,11 @@ import org.lucares.pdb.api.Tag;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.blockstorage.BSFile;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.PdbFile;
import org.lucares.pdb.datastore.PdbWriter;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.ReadException;
import org.lucares.pdb.datastore.WriteException;
import org.lucares.pdb.datastore.lang.Expression;
import org.lucares.pdb.datastore.lang.ExpressionToDocIdVisitor;
import org.lucares.pdb.datastore.lang.NewProposerParser;
@@ -42,6 +47,8 @@ public class DataStore implements AutoCloseable {
.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger MAP_DOCS_TO_DOCID = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.mapDocsToDocID");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.dataStore.newPdbWriter");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
public static final char LISTING_FILE_SEPARATOR = ',';
@@ -330,15 +337,14 @@ public class DataStore implements AutoCloseable {
return result;
}
public List<Doc> getByTags(final Tags tags) {
public Optional<Doc> getByTags(final Tags tags) {
try {
final Long docId = tagsToDocId.getValue(tags);
final List<Doc> result = new ArrayList<>(0);
if (docId != null) {
final Doc doc = getDocByDocId(docId);
result.add(doc);
return Optional.of(doc);
}
return result;
return Optional.empty();
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
@@ -375,4 +381,44 @@ public class DataStore implements AutoCloseable {
return diskStorage;
}
public PdbWriter getWriter(final Tags tags) {
final Optional<Doc> docsForTags = getByTags(tags);
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, getDiskStorage());
} catch (final IOException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(tags);
}
return writer;
}
private PdbWriter newPdbWriter(final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile, getDiskStorage());
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final IOException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final Tags tags) throws IOException {
final long rootBlockNumber = createNewFile(tags);
final PdbFile result = new PdbFile(rootBlockNumber, tags);
return result;
}
}

5
data-store/src/test/java/org/lucares/pdb/datastore/internal/DataStoreTest.java
View File

@@ -14,6 +14,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -110,8 +111,8 @@ public class DataStoreTest {
tagsToBlockStorageRootBlockNumber.put(pigeonJennifer, dataStore.createNewFile(pigeonJennifer));
tagsToBlockStorageRootBlockNumber.put(flamingoJennifer, dataStore.createNewFile(flamingoJennifer));
final List<Doc> docsFlamingoJennifer = dataStore.getByTags(flamingoJennifer);
Assert.assertEquals(docsFlamingoJennifer.size(), 1, "doc for docsFlamingoJennifer");
final Optional<Doc> docsFlamingoJennifer = dataStore.getByTags(flamingoJennifer);
Assert.assertTrue(docsFlamingoJennifer.isPresent(), "doc for docsFlamingoJennifer");
}
public void testBlockAlignment() throws IOException {

3
performanceDb/src/main/java/org/lucares/performance/db/Group.java
View File

@@ -3,6 +3,7 @@ package org.lucares.performance.db;
import java.util.List;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbFile;
class Group {
private final Tags tags;
@@ -29,6 +30,6 @@ class Group {
@Override
public String toString() {
return tags + ": " + files.size()+" files";
return tags + ": " + files.size() + " files";
}
}

1
performanceDb/src/main/java/org/lucares/performance/db/Grouping.java
View File

@@ -8,6 +8,7 @@ import java.util.List;
import java.util.Map;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

13
performanceDb/src/main/java/org/lucares/performance/db/PdbExport.java
View File

@@ -23,6 +23,7 @@ import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.config.Configurator;
import org.lucares.collections.LongList;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +83,7 @@ public class PdbExport {
for (final PdbFile pdbFile : pdbFiles) {
if (writer == null || Files.size(exportFile) > GB) {
if (writer == null || Files.size(exportFile) > 4 * GB) {
if (writer != null) {
writer.flush();
writer.close();
@@ -100,7 +101,6 @@ public class PdbExport {
final Tags tags = pdbFile.getTags();
final long tagsId = addNewTagsToDictionary(writer, tags, tagsIdCounter);
final Iterator<LongList> it = timeValueStream.iterator();
while (it.hasNext()) {
final LongList entry = it.next();
@@ -123,10 +123,11 @@ public class PdbExport {
count++;
final long chunk = 10_000_000;
if (count % chunk == 0) {
long end = System.currentTimeMillis();
long duration = end-begin;
long entriesPerSecond = (long)((double)chunk / (duration / 1000.0));
LOGGER.info("progress: {} - {} entries/s + duration {}" , String.format("%,d",count), String.format("%,d",entriesPerSecond), duration);
final long end = System.currentTimeMillis();
final long duration = end - begin;
final long entriesPerSecond = (long) (chunk / (duration / 1000.0));
LOGGER.info("progress: {} - {} entries/s + duration {}", String.format("%,d", count),
String.format("%,d", entriesPerSecond), duration);
begin = System.currentTimeMillis();
}
}

4
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -17,7 +17,11 @@ import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.InvalidValueException;
import org.lucares.pdb.datastore.PdbFile;
import org.lucares.pdb.datastore.PdbWriter;
import org.lucares.pdb.datastore.Proposal;
import org.lucares.pdb.datastore.WriteException;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.pdb.datastore.lang.SyntaxException;
import org.lucares.pdb.diskstorage.DiskStorage;

44
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -1,6 +1,5 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
@@ -8,6 +7,10 @@ import java.util.function.Consumer;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.Doc;
import org.lucares.pdb.datastore.PdbFile;
import org.lucares.pdb.datastore.PdbWriter;
import org.lucares.pdb.datastore.ReadException;
import org.lucares.pdb.datastore.WriteException;
import org.lucares.pdb.datastore.internal.DataStore;
import org.lucares.utils.cache.HotEntryCache;
import org.lucares.utils.cache.HotEntryCache.Event;
@@ -19,8 +22,6 @@ import org.slf4j.LoggerFactory;
public class TagsToFile implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(TagsToFile.class);
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.ingestion.tagsToFile.newPdbWriter");
private static final class CacheKey implements Comparable<CacheKey> {
private final Tags tags;
@@ -77,7 +78,6 @@ public class TagsToFile implements AutoCloseable {
writerCache = new HotEntryCache<>(Duration.ofSeconds(10), "writerCache");
writerCache.addListener(new RemovalListener(), EventType.EVICTED, EventType.REMOVED);
}
public List<PdbFile> getFilesForQuery(final String query) {
@@ -115,18 +115,7 @@ public class TagsToFile implements AutoCloseable {
if (writer == null) {
LOGGER.trace("getByTags({})", tags);
final List<Doc> docsForTags = dataStore.getByTags(tags);
if (docsForTags.size() > 0) {
try {
final Doc doc = docsForTags.get(0);
final PdbFile pdbFile = new PdbFile(doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, dataStore.getDiskStorage());
} catch (final IOException e) {
throw new ReadException(e);
}
} else {
writer = newPdbWriter(tags);
}
writer = dataStore.getWriter(tags);
writerCache.put(cacheKey, writer);
}
}
@@ -134,29 +123,6 @@ public class TagsToFile implements AutoCloseable {
return writer;
}
private PdbWriter newPdbWriter(final Tags tags) {
final long start = System.nanoTime();
try {
final PdbFile pdbFile = createNewPdbFile(tags);
final PdbWriter result = new PdbWriter(pdbFile, dataStore.getDiskStorage());
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final IOException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final Tags tags) throws IOException {
final long rootBlockNumber = dataStore.createNewFile(tags);
final PdbFile result = new PdbFile(rootBlockNumber, tags);
return result;
}
private void forEachWriter(final Consumer<PdbWriter> consumer) {
writerCache.forEach(writer -> {
try {

1
performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java
View File

@@ -8,6 +8,7 @@ import java.time.ZoneOffset;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.lucares.pdb.datastore.PdbWriter;
import org.lucares.pdb.datastore.internal.DataStore;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;