andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

reuse pdb writers

Browse Source
This commit is contained in:
Andreas Huber
2016-12-28 08:39:20 +01:00
parent db0b3d6d24
commit 68ac1dd631
8 changed files with 157 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
performanceDb/src/main/java/org/lucares/performance/db/FileSupplier.java Normal file
View File

@@ -0,0 +1,27 @@
package org.lucares.performance.db;
import java.io.IOException;
import java.time.OffsetDateTime;
import org.lucares.pdb.api.Tags;
import org.lucares.performance.db.PdbWriterManager.PdbFileSupplier;
class FileSupplier implements PdbFileSupplier {
private final TagsToFile tagsToFile;
public FileSupplier(final TagsToFile tagsToFile) {
super();
this.tagsToFile = tagsToFile;
}
@Override
public PdbFile supply(final Tags tags, final OffsetDateTime date) {
try {
final PdbFile pdbFile = tagsToFile.getFile(date, tags);
return pdbFile;
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}

6
performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java
View File

@@ -69,10 +69,4 @@ class PdbFile {
return false;
return true;
}
public TimeRange getTimeRange() {
// TODO @ahr should return the minimal date that can be added
return null;
}
}

20
performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java
View File

@@ -105,6 +105,7 @@ class PdbWriter implements AutoCloseable {
}
public void write(final Entry entry) throws WriteException {
System.out.println(entry);
final long epochMilli = entry.getEpochMilli();
final long value = entry.getValue();
write(epochMilli, value);
@@ -112,19 +113,9 @@ class PdbWriter implements AutoCloseable {
private void write(final long epochMilli, final long value) throws WriteException {
try {
if (epochMilli < lastEpochMilli) {
LOGGER.info("epochMilli must not be smaller than " + lastEpochMilli + ", but was " + epochMilli
+ ". We'll accept this for now. "
+ "Currently there is no code that relies on monotonically increasing date values. "
+ "Log4j does not guarantee it either.");
return;
}
final long epochMilliIncrement = epochMilli - lastEpochMilli;
assertValueInRange(epochMilliIncrement);
assertValueInRange(value);
assertEpochMilliInRange(epochMilli);
writeValue(epochMilliIncrement, ByteType.DATE_INCREMENT, outputStream);
writeValue(value, ByteType.MEASUREMENT, outputStream);
@@ -134,15 +125,6 @@ class PdbWriter implements AutoCloseable {
}
}
private void assertEpochMilliInRange(final long epochMilli) {
if (epochMilli < lastEpochMilli) {
LOGGER.info("epochMilli must not be smaller than " + lastEpochMilli + ", but was " + epochMilli
+ ". We'll accept this for now. "
+ "Currently there is no code that relies on monotonically increasing date values. "
+ "Log4j does not guarantee it either.");
}
}
private void assertValueInRange(final long value) {
if (value < 0) {
throw new IllegalArgumentException("value must not be negative: " + value);

74
performanceDb/src/main/java/org/lucares/performance/db/PdbWriterManager.java
View File

@@ -15,72 +15,31 @@ public class PdbWriterManager implements AutoCloseable {
private final static Logger LOGGER = Logger.getLogger(PdbWriterManager.class.getCanonicalName());
private final static class Key {
private final Tags tags;
private final Day day;
public Key(final Tags tags, final OffsetDateTime date) {
super();
this.tags = tags;
this.day = new Day(date);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((day == null) ? 0 : day.hashCode());
result = prime * result + ((tags == null) ? 0 : tags.hashCode());
return result;
}
@Override
public boolean equals(final Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
final Key other = (Key) obj;
if (day == null) {
if (other.day != null)
return false;
} else if (!day.equals(other.day))
return false;
if (tags == null) {
if (other.tags != null)
return false;
} else if (!tags.equals(other.tags))
return false;
return true;
}
public interface PdbFileSupplier {
public PdbFile supply(Tags tags, OffsetDateTime date);
}
public interface PdbWriterSupplier {
public PdbWriter supply(Tags tags, OffsetDateTime date);
}
final Map<PdbFile, PdbWriter> map = new HashMap<>();
final Map<Key, PdbWriter> map = new HashMap<>();
private final PdbWriterSupplier supplier;
private final PdbFileSupplier supplier;
private Day lastDay = new Day(OffsetDateTime.MIN);
public PdbWriterManager(final PdbWriterSupplier supplier) {
public PdbWriterManager(final PdbFileSupplier supplier) {
this.supplier = supplier;
}
public PdbWriter get(final Tags tags, final OffsetDateTime date) {
public PdbWriter getWriter(final Tags tags, final OffsetDateTime date) throws IOException {
handleDateChange(date);
final Key key = new Key(tags, date);
if (!map.containsKey(key)) {
final PdbWriter writer = supplier.supply(tags, date);
put(tags, date, writer);
final PdbFile pdbFile = supplier.supply(tags, date);
if (!map.containsKey(pdbFile)) {
final PdbWriter writer = new PdbWriter(pdbFile);
map.put(pdbFile, writer);
}
return map.get(key);
return map.get(pdbFile);
}
private void handleDateChange(final OffsetDateTime date) {
@@ -94,11 +53,6 @@ public class PdbWriterManager implements AutoCloseable {
}
}
public PdbWriter put(final Tags tags, final OffsetDateTime date, final PdbWriter pdbWriter) {
final Key key = new Key(tags, date);
return map.put(key, pdbWriter);
}
public void flush() {
LOGGER.info("flushing all files");
for (final PdbWriter writer : map.values()) {
@@ -116,10 +70,10 @@ public class PdbWriterManager implements AutoCloseable {
}
private void closeFiles() {
final Iterator<Entry<Key, PdbWriter>> it = map.entrySet().iterator();
final Iterator<Entry<PdbFile, PdbWriter>> it = map.entrySet().iterator();
while (it.hasNext()) {
final Entry<Key, PdbWriter> entry = it.next();
final Entry<PdbFile, PdbWriter> entry = it.next();
final PdbWriter writer = entry.getValue();
try {
writer.close();

29
performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java
View File

@@ -22,7 +22,6 @@ import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.GroupResult;
import org.lucares.pdb.api.Result;
import org.lucares.pdb.api.Tags;
import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
public class PerformanceDb implements AutoCloseable {
private static final Logger LOGGER = Logger.getLogger(PerformanceDb.class.getCanonicalName());
@@ -57,34 +56,12 @@ public class PerformanceDb implements AutoCloseable {
put(iterator);
}
private static class WriterSupplier implements PdbWriterSupplier {
private final TagsToFile tagsToFile;
public WriterSupplier(final TagsToFile tagsToFile) {
super();
this.tagsToFile = tagsToFile;
}
@Override
public PdbWriter supply(final Tags tags, final OffsetDateTime date) {
try {
final PdbFile pdbFile = tagsToFile.getFile(date, tags);
final PdbWriter writer = new PdbWriter(pdbFile);
return writer;
} catch (final IOException e) {
throw new RuntimeException(e);
}
}
}
public void put(final BlockingIterator<Entry> entries) throws WriteException {
long count = 0;
double durationInManager = 0;
try (final PdbWriterManager manager = new PdbWriterManager(new WriterSupplier(tagsToFile));) {
try (final PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
long start = System.nanoTime();
while (true) {
@@ -98,7 +75,7 @@ public class PerformanceDb implements AutoCloseable {
final Tags tags = entry.getTags();
final OffsetDateTime date = entry.getDate();
final PdbWriter writer = manager.get(tags, date);
final PdbWriter writer = manager.getWriter(tags, date);
writer.write(entry);
count++;
@@ -117,6 +94,8 @@ public class PerformanceDb implements AutoCloseable {
}
}
} catch (final IOException e) {
throw new WriteException(e);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
LOGGER.info("Thread was interrupted. Aborting exectution.");

69
performanceDb/src/test/java/org/lucares/performance/db/PdbWriterManagerTest.java
View File

@@ -4,9 +4,11 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.lucares.ludb.H2DB;
import org.lucares.pdb.api.Entry;
import org.lucares.pdb.api.Tags;
import org.lucares.performance.db.PdbWriterManager.PdbWriterSupplier;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -29,31 +31,52 @@ public class PdbWriterManagerTest {
@Test
public void testManager() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final PdbWriterSupplier supplier = (tags, date) -> {
Path path;
try {
path = Files.createTempFile(dataDirectory, "pdb", ".data");
return new PdbWriter(new PdbFile(path, tags));
} catch (final IOException e) {
throw new AssertionError(e.getMessage(), e);
final Tags tagsA = Tags.create("key", "A");
final Tags tagsB = Tags.create("key", "B");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime date = OffsetDateTime.now();
final PdbWriter firstWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter secondWriterForTagsA = manager.getWriter(tagsA, date);
final PdbWriter firstWriterForTagsB = manager.getWriter(tagsB, date);
Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
}
};
final Tags tagsA = Tags.create("key", "A");
final Tags tagsB = Tags.create("key", "B");
try (PdbWriterManager manager = new PdbWriterManager(supplier)) {
final OffsetDateTime date = OffsetDateTime.now();
final PdbWriter firstWriterForTagsA = manager.get(tagsA, date);
final PdbWriter secondWriterForTagsA = manager.get(tagsA, date);
final PdbWriter firstWriterForTagsB = manager.get(tagsB, date);
Assert.assertSame(firstWriterForTagsA, secondWriterForTagsA);
Assert.assertNotSame(firstWriterForTagsA, firstWriterForTagsB);
}
}
@Test
public void testManager2() throws Exception {
try (H2DB db = new H2DB(dataDirectory.toFile())) {
final TagsToFile tagsToFile = new TagsToFile(dataDirectory, db);
final Tags tags = Tags.create("key", "A");
try (PdbWriterManager manager = new PdbWriterManager(new FileSupplier(tagsToFile))) {
final OffsetDateTime morning = OffsetDateTime.of(2016, 1, 1, 8, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime noon = OffsetDateTime.of(2016, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC);
final OffsetDateTime afternoon = OffsetDateTime.of(2016, 1, 1, 17, 0, 0, 0, ZoneOffset.UTC);
final PdbWriter writerNoon = manager.getWriter(tags, noon);
writerNoon.write(new Entry(noon, 1, tags));
final PdbWriter writerMorning = manager.getWriter(tags, morning);
writerMorning.write(new Entry(morning, 2, tags));
final PdbWriter writerAfternoon = manager.getWriter(tags, afternoon);
writerAfternoon.write(new Entry(afternoon, 3, tags));
Assert.assertSame(writerNoon, writerAfternoon);
Assert.assertNotSame(writerNoon, writerMorning);
}
}
}
}