andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

split BSFile into a TimeSeries and a LongStream file

Browse Source 
BSFile was used to store two types of data. This makes
the API complex. I split the API into two files with
easier and more clear APIs. Interestingly the API of
BSFile is still rather complex and has to consider both
use cases.
This commit is contained in:
Andreas Huber
2019-02-10 09:59:16 +01:00
parent fd55ea0866
commit cbcb7714bb
16 changed files with 546 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
block-storage/src/test/java/org/lucares/pdb/blockstorage/BSFileTest.java
View File

@@ -8,7 +8,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -47,7 +46,7 @@ public class BSFileTest {
//
try (final DiskStorage ds = new DiskStorage(file)) {
try (final BSFile bsFile = BSFile.newFile(ds)) {
try (final BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
blockOffset = bsFile.getRootBlockOffset();
@@ -55,7 +54,7 @@ public class BSFileTest {
bsFile.append(i);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds)) {
try (final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE)) {
for (long i = numLongs / 2; i < numLongs; i++) {
bsFile.append(i);
@@ -66,7 +65,7 @@ public class BSFileTest {
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
final BSFile bsFile = BSFile.existingFile(blockOffset, ds);
final BSFile bsFile = BSFile.existingFile(blockOffset, ds, NullCustomizer.INSTANCE);
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = LongList.rangeClosed(0, numLongs - 1);
Assert.assertEquals(actualLongs, expectedLongs);
@@ -91,7 +90,7 @@ public class BSFileTest {
final ThreadLocalRandom random = ThreadLocalRandom.current();
final LongList listOfValues = new LongList();
try (BSFile bsFile = BSFile.newFile(ds)) {
try (BSFile bsFile = BSFile.newFile(ds, NullCustomizer.INSTANCE)) {
for (int j = 0; j < values; j++) {
@@ -123,7 +122,7 @@ public class BSFileTest {
final long rootBlockNumber = entry.getKey();
final LongList expectedValues = entry.getValue();
try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds)) {
try (BSFile bsFile = BSFile.existingFile(rootBlockNumber, ds, NullCustomizer.INSTANCE)) {
final LongList actualLongs = bsFile.asLongList();
final LongList expectedLongs = expectedValues;
Assert.assertEquals(actualLongs, expectedLongs, "for rootBlockNumber=" + rootBlockNumber);
@@ -132,58 +131,4 @@ public class BSFileTest {
}
}
public void testBlockStorageTimeValue() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final Random random = ThreadLocalRandom.current();
final int numTimeValuePairs = 1000;
long blockNumber = -1;
final LongList expectedLongs = new LongList();
long start = System.nanoTime();
long lastEpochMilli = 0;
//
try (final DiskStorage ds = new DiskStorage(file)) {
try (final BSFile bsFile = BSFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(1000);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
try (final BSFile bsFile = BSFile.existingFile(blockNumber, ds)) {
for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(100);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
final BSFile bsFile = BSFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList();
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}

84
block-storage/src/test/java/org/lucares/pdb/blockstorage/TimeSeriesFileTest.java Normal file
View File

@@ -0,0 +1,84 @@
package org.lucares.pdb.blockstorage;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import org.lucares.collections.LongList;
import org.lucares.pdb.diskstorage.DiskStorage;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
public class TimeSeriesFileTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void testBlockStorageTimeValue() throws Exception {
final Path file = dataDirectory.resolve("data.int.db");
final Random random = ThreadLocalRandom.current();
final int numTimeValuePairs = 1000;
long blockNumber = -1;
final LongList expectedLongs = new LongList();
long start = System.nanoTime();
long lastEpochMilli = 0;
//
try (final DiskStorage ds = new DiskStorage(file)) {
try (final TimeSeriesFile bsFile = TimeSeriesFile.newFile(ds)) {
blockNumber = bsFile.getRootBlockOffset();
for (long i = 0; i < numTimeValuePairs / 2; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(1000);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
try (final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds)) {
for (long i = numTimeValuePairs / 2; i < numTimeValuePairs; i++) {
final long epochMilli = lastEpochMilli + random.nextInt(100);
final long value = random.nextInt(10000);
lastEpochMilli = epochMilli;
bsFile.appendTimeValue(epochMilli, value);
expectedLongs.add(epochMilli);
expectedLongs.add(value);
}
}
}
System.out.println("duration write: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
start = System.nanoTime();
try (final DiskStorage ds = new DiskStorage(file)) {
final TimeSeriesFile bsFile = TimeSeriesFile.existingFile(blockNumber, ds);
final LongList actualLongs = bsFile.asTimeValueLongList();
Assert.assertEquals(actualLongs, expectedLongs);
}
System.out.println("duration read: " + (System.nanoTime() - start) / 1_000_000.0 + "ms");
}
}

124
block-storage/src/test/java/org/lucares/pdb/map/PersistentMapOfListsOfLongsTest.java
View File

@@ -1,62 +1,62 @@
package org.lucares.pdb.map;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.lucares.collections.LongList;
import org.lucares.utils.file.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@Test
public class PersistentMapOfListsOfLongsTest {
private Path dataDirectory;
@BeforeMethod
public void beforeMethod() throws IOException {
dataDirectory = Files.createTempDirectory("pdb");
}
@AfterMethod
public void afterMethod() throws IOException {
FileUtils.delete(dataDirectory);
}
public void test() throws IOException {
final String mapFilePrefix = "test";
final String keyA = "a";
final String keyB = "b";
final int size = 10;
final LongList a = LongList.range(0, size);
a.shuffle();
final LongList b = LongList.range(0, size);
b.shuffle();
try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
PersistentMap.STRING_CODER)) {
for (int i = 0; i < size; i++) {
map.appendLong(keyA, a.get(i));
map.appendLong(keyB, b.get(i));
}
}
try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
PersistentMap.STRING_CODER)) {
final LongList actualA = new LongList();
map.getLongs(keyA).forEachOrdered(actualA::addAll);
Assert.assertEquals(actualA, a);
final LongList actualB = new LongList();
map.getLongs(keyB).forEachOrdered(actualB::addAll);
Assert.assertEquals(actualB, b);
}
}
}
//package org.lucares.pdb.map;
//
//import java.io.IOException;
//import java.nio.file.Files;
//import java.nio.file.Path;
//
//import org.lucares.collections.LongList;
//import org.lucares.utils.file.FileUtils;
//import org.testng.Assert;
//import org.testng.annotations.AfterMethod;
//import org.testng.annotations.BeforeMethod;
//import org.testng.annotations.Test;
//
//@Test
//public class PersistentMapOfListsOfLongsTest {
//
// private Path dataDirectory;
//
// @BeforeMethod
// public void beforeMethod() throws IOException {
// dataDirectory = Files.createTempDirectory("pdb");
// }
//
// @AfterMethod
// public void afterMethod() throws IOException {
// FileUtils.delete(dataDirectory);
// }
//
// public void test() throws IOException {
//
// final String mapFilePrefix = "test";
// final String keyA = "a";
// final String keyB = "b";
//
// final int size = 10;
// final LongList a = LongList.range(0, size);
// a.shuffle();
// final LongList b = LongList.range(0, size);
// b.shuffle();
//
// try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
// PersistentMap.STRING_CODER)) {
//
// for (int i = 0; i < size; i++) {
// map.appendLong(keyA, a.get(i));
// map.appendLong(keyB, b.get(i));
// }
// }
//
// try (PersistentMapOfListsOfLongs<String> map = new PersistentMapOfListsOfLongs<>(dataDirectory, mapFilePrefix,
// PersistentMap.STRING_CODER)) {
//
// final LongList actualA = new LongList();
// map.getLongs(keyA).forEachOrdered(actualA::addAll);
// Assert.assertEquals(actualA, a);
//
// final LongList actualB = new LongList();
// map.getLongs(keyB).forEachOrdered(actualB::addAll);
// Assert.assertEquals(actualB, b);
// }
// }
//}