andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

reduce memory usage

Browse Source 
Reduce memory usage by storing the filename as string instead of
individual tags.
This commit is contained in:
Andreas Huber
2018-03-19 19:21:57 +01:00
parent 181fce805d
commit 5343c0d427
20 changed files with 315 additions and 454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -3,7 +3,6 @@ package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -94,9 +93,8 @@ public class PerformanceDb implements AutoCloseable {
final long duration = end - lastSync;
final long entriesPerSecond = (long) (insertionsSinceLastSync / (duration / 1000.0));
METRICS_LOGGER
.debug(String.format("inserting %d/s ; total: %,d; last: %s",
entriesPerSecond, count, entry));
METRICS_LOGGER.debug(
String.format("inserting %d/s ; total: %,d; last: %s", entriesPerSecond, count, entry));
tagsToFile.flush();
lastSync = System.currentTimeMillis();
@@ -156,8 +154,8 @@ public class PerformanceDb implements AutoCloseable {
final Grouping grouping = Grouping.groupBy(pdbFiles, groupBy);
final Result result = toResult(grouping);
METRICS_LOGGER.debug("query execution took: " + (System.nanoTime() - start) / 1_000_000.0
+ "ms: " + query + " ("+groupBy+"): files found: " + pdbFiles.size());
METRICS_LOGGER.debug("query execution took: " + (System.nanoTime() - start) / 1_000_000.0 + "ms: " + query
+ " (" + groupBy + "): files found: " + pdbFiles.size());
return result;
}

23
performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java
View File

@@ -2,7 +2,6 @@ package org.lucares.performance.db;
import java.io.IOException;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -24,8 +23,8 @@ import org.slf4j.LoggerFactory;
public class TagsToFile implements AutoCloseable {
private static final Logger LOGGER = LoggerFactory.getLogger(TagsToFile.class);
private final static Logger METRICS_LOGGER_FIND_WRITER = LoggerFactory.getLogger("org.lucares.metrics.ingestion.tagsToFile.findWriter");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory.getLogger("org.lucares.metrics.ingestion.tagsToFile.newPdbWriter");
private final static Logger METRICS_LOGGER_NEW_WRITER = LoggerFactory
.getLogger("org.lucares.metrics.ingestion.tagsToFile.newPdbWriter");
private static class WriterCache {
final List<PdbWriter> writers = new ArrayList<>();
@@ -57,9 +56,9 @@ public class TagsToFile implements AutoCloseable {
}
public List<PdbFile> getFilesForQuery(final String query) {
final List<Doc> searchResult = db.search(query);
if (searchResult.size() > 500_000){
if (searchResult.size() > 500_000) {
throw new IllegalStateException("Too many results.");
}
@@ -84,7 +83,8 @@ public class TagsToFile implements AutoCloseable {
final PdbWriter result;
final WriterCache writersForTags = getOrInit(tags);
final Optional<PdbWriter> optionalWriter = chooseBestMatchingWriter(writersForTags.getWriters(), dateAsEpochMilli);
final Optional<PdbWriter> optionalWriter = chooseBestMatchingWriter(writersForTags.getWriters(),
dateAsEpochMilli);
if (optionalWriter.isPresent()) {
result = optionalWriter.get();
@@ -97,7 +97,7 @@ public class TagsToFile implements AutoCloseable {
final List<Optional<PdbWriter>> existingWriters = CollectionUtils.filter(optionalWriters,
Optional::isPresent);
final List<PdbWriter> writers = CollectionUtils.map(existingWriters, Optional::get);
final Optional<PdbWriter> optionalFirst = chooseBestMatchingWriter(writers, dateAsEpochMilli);
result = optionalFirst.orElseGet(() -> newPdbWriter(tags));
@@ -161,14 +161,15 @@ public class TagsToFile implements AutoCloseable {
final PdbWriter result = new PdbWriter(pdbFile);
getOrInit(tags).addWriter(result);
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}", (System.nanoTime() - start) / 1_000_000.0, tags);
METRICS_LOGGER_NEW_WRITER.debug("newPdbWriter took {}ms tags: {}",
(System.nanoTime() - start) / 1_000_000.0, tags);
return result;
} catch (final IOException e) {
throw new WriteException(e);
}
}
private PdbFile createNewPdbFile(final Tags tags) throws IOException {