andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add second parser that uses a standard CSV reader

Browse Source
This commit is contained in:
Andreas Huber
2021-08-12 17:54:27 +02:00
parent 825bac24b9
commit 67c66ef89d
18 changed files with 584 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -159,7 +159,13 @@ public class DataStore implements AutoCloseable {
public void write(final long dateAsEpochMilli, final Tags tags, final long value) {
final ParititionId partitionId = DateIndexExtension.toPartitionId(dateAsEpochMilli);
final PdbWriter writer = getWriter(partitionId, tags);
final long start = System.nanoTime();
writer.write(dateAsEpochMilli, value);
final double duration = (System.nanoTime() - start) / 1_000_000.0;
if (duration > 1) {
System.out.println(" write took: " + duration + " ms " + tags);
}
}
// visible for test
@@ -377,9 +383,14 @@ public class DataStore implements AutoCloseable {
}
private PdbWriter getWriter(final ParititionId partitionId, final Tags tags) throws ReadException, WriteException {
final long start = System.nanoTime();
final PartitionedTagsCacheKey cacheKey = new PartitionedTagsCacheKey(tags, partitionId);
return writerCache.putIfAbsent(cacheKey, t -> getWriterInternal(partitionId, tags));
final PdbWriter result = writerCache.putIfAbsent(cacheKey, t -> getWriterInternal(partitionId, tags));
final double duration = (System.nanoTime() - start) / 1_000_000.0;
if (duration > 1) {
System.out.println(" get Writer took: " + duration + " ms " + tags);
}
return result;
}
// visible for test
@@ -392,9 +403,14 @@ public class DataStore implements AutoCloseable {
PdbWriter writer;
if (docsForTags.isPresent()) {
try {
final long start = System.nanoTime();
final Doc doc = docsForTags.get();
final PdbFile pdbFile = new PdbFile(partitionId, doc.getRootBlockNumber(), tags);
writer = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
final double duration = (System.nanoTime() - start) / 1_000_000.0;
if (duration > 1) {
System.out.println(" init existing writer took: " + duration + " ms " + tags);
}
} catch (final RuntimeException e) {
throw new ReadException(e);
}
@@ -410,8 +426,10 @@ public class DataStore implements AutoCloseable {
final PdbFile pdbFile = createNewPdbFile(partitionId, tags);
final PdbWriter result = new PdbWriter(pdbFile, diskStorage.getExisting(partitionId));
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
final double duration = (System.nanoTime() - start) / 1_000_000.0;
if (duration > 1) {
METRICS_LOGGER_NEW_WRITER.info("newPdbWriter took {}ms tags: {}", duration, tags);
}
return result;
} catch (final RuntimeException e) {
throw new WriteException(e);

5
data-store/src/main/java/org/lucares/pdb/datastore/internal/QueryCompletionIndex.java
View File

@@ -313,6 +313,7 @@ public class QueryCompletionIndex implements AutoCloseable {
}
public void addTags(final ParititionId partitionId, final Tags tags) throws IOException {
final long start = System.nanoTime();
final List<Tag> listOfTagsA = tags.toTags();
final List<Tag> listOfTagsB = tags.toTags();
@@ -329,6 +330,10 @@ public class QueryCompletionIndex implements AutoCloseable {
fieldToValueIndex.putValue(partitionId, tag, Empty.INSTANCE);
fieldIndex.putValue(partitionId, Tags.STRING_COMPRESSOR.getKeyAsString(tag), Empty.INSTANCE);
}
final double d = (System.nanoTime() - start) / 1_000_000.0;
if (d > 1) {
System.out.println(" addTags: " + d + " ms");
}
}
@Override