andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

reduce memory footprint of index by trimming IntLists

Browse Source 
Reduced the memory usage of the IntLists in the index by 4.1MB (19.9MB
to 15.8MB) for 683,390 files and 4,046,250 values in the IntLists.
This commit is contained in:
ahr
2017-12-16 17:57:15 +01:00
parent 8225dd2077
commit 6ef4e7a96b
1 changed files with 26 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
data-store/src/main/java/org/lucares/pdb/datastore/internal/DataStore.java
View File

@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
public class DataStore {
private static final Logger EXECUTE_QUERY_LOGGER = LoggerFactory.getLogger("org.lucares.metrics.dataStore.executeQuery");
private static final Logger LOGGER = LoggerFactory.getLogger(DataStore.class);
private static final String SUBDIR_STORAGE = "storage";
private static final String PDB_EXTENSION = ".pdb";
@@ -68,6 +69,7 @@ public class DataStore {
cacheTagToFileMapping(tags, path);
});
trimIntLists();
}
private void cacheTagToFileMapping(final Tags tags, final Path path) {
@@ -85,6 +87,30 @@ public class DataStore {
}
}
private void trimIntLists() {
final long start = System.nanoTime();
int totalBeforeTrim = 0;
int totalAfterTrim = 0;
int totalValues = 0;
for (Map<String, IntList> valueToDocIds : keyToValueToDocId.values()) {
for (IntList intList : valueToDocIds.values()) {
totalBeforeTrim += intList.getCapacity();
intList.trim();
totalAfterTrim += intList.getCapacity();
totalValues += intList.size();
}
}
LOGGER.info(
"trimming IntLists of index: values {}, {} kB before, {} kB after, difference {} kB, total size: {} kB, took: {} ms",
totalValues,
(totalBeforeTrim * 4) / 1024, (totalAfterTrim * 4) / 1024,
((totalBeforeTrim - totalAfterTrim) * 4) / 1024,
(totalValues * 4) / 1024,
(System.nanoTime() - start) / 1_000_000.0);
}
private Path keyCompressionFile(final Path dataDirectory) throws IOException {
return dataDirectory.resolve("keys.csv");
}