commit 256b2784287fd7a7f588827476cb36e2cfb50b8f Author: Andreas Huber Date: Sun Dec 4 10:41:37 2016 +0100 inital commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f1d628f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/.settings/ +/.classpath +/.project +/.gradle/ +/build/ diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..37aa70c --- /dev/null +++ b/build.gradle @@ -0,0 +1,40 @@ +apply plugin: 'java' +apply plugin: 'eclipse' + +/* + * The shared configuration for all sub-projects: + */ +subprojects { + apply plugin: 'java' + apply plugin: 'eclipse' + + // java compatibility version + sourceCompatibility = 1.8 + + + configurations { + tests + } + + // the repositories for external depenencies + repositories { + maven { url 'http://repo.lucares.de/' } + mavenCentral() + jcenter() + } + + // In this example we use TestNG as our testing tool. JUnit is the default. + test{ + useTestNG() + } + + // dependencies that all sub-projects have + dependencies { + testCompile group: 'org.testng', name: 'testng', version: '6.9.6' + //testCompile group: 'junit', name: 'junit', version: '4.12' + } +} + +task wrapper(type: Wrapper) { + gradleVersion = '2.12' +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..b761216 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..97e04b2 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Sat Sep 13 09:45:49 CEST 2014 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-2.12-bin.zip diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..91a7e26 --- /dev/null +++ b/gradlew @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# For Cygwin, ensure paths are in UNIX format before anything is touched. +if $cygwin ; then + [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` +fi + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >&- +APP_HOME="`pwd -P`" +cd "$SAVED" >&- + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..aec9973 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windowz variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/gradlew.sh b/gradlew.sh new file mode 100755 index 0000000..79238e3 --- /dev/null +++ b/gradlew.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +mkdir -p tmp +cd tmp + +if ! test -d "gradle-2.0"; then + wget https://services.gradle.org/distributions/gradle-2.0-bin.zip + unzip gradle-2.0-bin.zip +fi + + +#echo setting environment + +export GRADLE_HOME=`pwd`/gradle-2.0 +#echo GRADLE_HOME=$GRADLE_HOME + +export PATH=$PATH:$GRADLE_HOME/bin +#echo $PATH + + +cd .. +gradle --daemon $1 $2 $3 $4 $5 $6 $7 diff --git a/performanceDb/.gitignore b/performanceDb/.gitignore new file mode 100644 index 0000000..0e895c8 --- /dev/null +++ b/performanceDb/.gitignore @@ -0,0 +1,6 @@ +/bin/ +/build/ +/.settings/ +/.classpath +/.project +/test-output diff --git a/performanceDb/build.gradle b/performanceDb/build.gradle new file mode 100644 index 0000000..a7fa47a --- /dev/null +++ b/performanceDb/build.gradle @@ -0,0 +1,7 @@ + +dependencies { + //compile 'com.fasterxml.jackson.core:jackson-databind:2.8.5' + compile 'org.lucares:ludb:1.0.20161002111352' + //compile 'commons-io:commons-io:2.5' +} + diff --git a/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java b/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java new file mode 100644 index 0000000..2ed0aef --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/BitFiddling.java @@ -0,0 +1,27 @@ +package org.lucares.performance.db; + +class BitFiddling { + + static byte long3(final long x) { + return (byte) (x >> 24); + } + + static byte long2(final long x) { + return (byte) (x >> 16); + } + + static byte long1(final long x) { + return (byte) (x >> 8); + } + + static byte long0(final long x) { + return (byte) (x); + } + + static long makeLong(final byte b3, final byte b2, final byte b1, final byte b0) { + return ((((long) b3 & 0xff) << 24) | // + (((long) b2 & 0xff) << 16) | // + (((long) b1 & 0xff) << 8) | // + (((long) b0 & 0xff)));// + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java new file mode 100644 index 0000000..8382070 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/CollectionUtils.java @@ -0,0 +1,27 @@ +package org.lucares.performance.db; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +public interface CollectionUtils { + + public default Map toMap(final Iterable iterable, final Function keyMapper) { + final Map result = new HashMap<>(); + + for (final V value : iterable) { + final T key = keyMapper.apply(value); + + result.put(key, value); + } + + return result; + } + + public default List filter(final List list, final Predicate predicate) { + return list.stream().filter(predicate).collect(Collectors.toList()); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java new file mode 100644 index 0000000..f1c221f --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/DateUtils.java @@ -0,0 +1,83 @@ +package org.lucares.performance.db; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.TimeZone; + +public class DateUtils { + + private static final TimeZone TIME_ZONE_UTC = TimeZone.getTimeZone("UTC"); + private static final SimpleDateFormat YEAR = new SimpleDateFormat("yyyy"); + private static final SimpleDateFormat MONTH = new SimpleDateFormat("MM"); + private static final SimpleDateFormat DAY = new SimpleDateFormat("dd"); + + public static final synchronized Day getDay(final Date date) { + + final String yearString = YEAR.format(date); + final String monthString = MONTH.format(date); + final String dayString = DAY.format(date); + + final int year = Integer.parseInt(yearString, 10); + final int month = Integer.parseInt(monthString, 10); + final int day = Integer.parseInt(dayString, 10); + + return new Day(year, month, day); + } + + public static long getDateOffset(final Date date) { + + return getMidnightSameDay(date).getTime(); + } + + public static Calendar getCalendar() { + return Calendar.getInstance(TIME_ZONE_UTC); + } + + public static Date getMidnightSameDay(final Date date) { + + final Calendar exactTime = getCalendar(); + exactTime.setTime(date); + + final Calendar midnight = getCalendar(); + final int year = exactTime.get(Calendar.YEAR); + final int month = exactTime.get(Calendar.MONTH); + final int day = exactTime.get(Calendar.DATE); + midnight.set(year, month, day); + midnight.set(Calendar.HOUR, 0); + midnight.set(Calendar.MINUTE, 0); + midnight.set(Calendar.SECOND, 0); + midnight.set(Calendar.MILLISECOND, 0); + + return midnight.getTime(); + } + + public static Date getMidnightNextDay(final Date date) { + final Calendar exactTime = Calendar.getInstance(TIME_ZONE_UTC); + exactTime.setTime(date); + + exactTime.add(Calendar.DATE, 1); + exactTime.add(Calendar.MILLISECOND, -1); + + return exactTime.getTime(); + } + + public static Date getDate(final int year, final int month, final int day, final int hour, final int minute, + final int second) { + + final Calendar calendar = getCalendar(); + calendar.set(year, month - 1, day); + calendar.set(Calendar.HOUR, hour); + calendar.set(Calendar.MINUTE, minute); + calendar.set(Calendar.SECOND, second); + calendar.set(Calendar.MILLISECOND, 0); + return calendar.getTime(); + } + + public static String format(final Date date) { + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH-mm:ss,SSS Z"); + dateFormat.setTimeZone(TIME_ZONE_UTC); + + return dateFormat.format(date); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Day.java b/performanceDb/src/main/java/org/lucares/performance/db/Day.java new file mode 100644 index 0000000..1a499f9 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/Day.java @@ -0,0 +1,36 @@ +package org.lucares.performance.db; + +class Day { + private final int year; + private final int month; + private final int day; + + public Day(final int year, final int month, final int day) { + super(); + this.year = year; + this.month = month; + this.day = day; + } + + public int getYear() { + return year; + } + + public int getMonth() { + return month; + } + + public int getDay() { + return day; + } + + @Override + public String toString() { + return format("-"); + } + + public String format(final String separator) { + return String.format("%04d%s%02d%s%02d", year, separator, month, separator, day); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Entry.java b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java new file mode 100644 index 0000000..82ba180 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/Entry.java @@ -0,0 +1,56 @@ +package org.lucares.performance.db; + +import java.util.Date; + +public class Entry { + private final Date date; + + private final long value; + + public Entry(final Date date, final long value) { + super(); + this.date = date; + this.value = value; + } + + public Date getDate() { + return date; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return DateUtils.format(date) + " = " + value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((date == null) ? 0 : date.hashCode()); + result = prime * result + (int) (value ^ (value >>> 32)); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Entry other = (Entry) obj; + if (date == null) { + if (other.date != null) + return false; + } else if (!date.equals(other.date)) + return false; + if (value != other.value) + return false; + return true; + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Fields.java b/performanceDb/src/main/java/org/lucares/performance/db/Fields.java new file mode 100644 index 0000000..224331b --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/Fields.java @@ -0,0 +1,23 @@ +package org.lucares.performance.db; + +class Fields { + static final String TAG_PREFIX = "__tag_"; + + static final String DATE_OFFSET = "__date_offset__"; + + static String prefixedKey(final String key) { + return TAG_PREFIX + key; + } + + static boolean isPrefixedKey(final String key) { + return key.startsWith(TAG_PREFIX); + } + + static String stripPrefix(final String key) { + if (!isPrefixedKey(key)) { + throw new IllegalArgumentException(key + " is not prefixed by " + TAG_PREFIX); + } + return key.substring(TAG_PREFIX.length()); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java new file mode 100644 index 0000000..e9540ab --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/FileUtils.java @@ -0,0 +1,55 @@ +package org.lucares.performance.db; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class FileUtils { + private static final Logger LOGGER = Logger.getLogger(FileUtils.class.getCanonicalName()); + + private static final class RecursiveDeleter extends SimpleFileVisitor { + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + + Files.delete(file); + LOGGER.info("deleted: " + file.toFile().getAbsolutePath()); + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + + Files.delete(dir); + LOGGER.info("deleted: " + dir.toFile().getAbsolutePath()); + + return FileVisitResult.CONTINUE; + } + } + + public static void delete(final Path path) { + + final int maxAttempts = 10; + int attempt = 1; + + while (attempt <= maxAttempts) { + try { + LOGGER.info( + "deleting '" + path.toFile().getAbsolutePath() + "' attempt " + attempt + " of " + maxAttempts); + Files.walkFileTree(path, new RecursiveDeleter()); + break; + } catch (final IOException e) { + final String msg = "failed to delete '" + path.toFile().getAbsolutePath() + "' on attempt " + attempt + + " of " + maxAttempts; + LOGGER.log(Level.WARNING, msg, e); + } + attempt++; + } + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java new file mode 100644 index 0000000..10a7985 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFile.java @@ -0,0 +1,91 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.util.Date; + +class PdbFile { + private final Tags tags; + + private final long dateOffset; + + private final File file; + + public PdbFile(final long dateOffset, final File file, final Tags tags) { + checkOffset(dateOffset); + this.dateOffset = dateOffset; + this.file = file; + this.tags = tags; + } + + public static PdbFile today(final File file, final Tags tags) { + final long dateOffset = DateUtils.getDateOffset(new Date()); + return new PdbFile(dateOffset, file, tags); + } + + private void checkOffset(final long dateOffset) { + + final Date date = new Date(dateOffset); + final long expectedDateOffset = DateUtils.getDateOffset(date); + if (dateOffset != expectedDateOffset) { + throw new IllegalArgumentException("dateOffset must be at exactly midnight UTC: " + dateOffset + " != " + + expectedDateOffset + "(" + date + " != " + new Date(expectedDateOffset) + ")"); + } + } + + public Tags getTags() { + return tags; + } + + public File getFile() { + return file; + } + + public long getDateOffset() { + return dateOffset; + } + + public TimeRange getTimeRange() { + final Date from = new Date(dateOffset); + final Date to = DateUtils.getMidnightNextDay(from); + return new TimeRange(from, to); + } + + @Override + public String toString() { + return "PdbFile [" + file + " " + getTimeRange() + " " + tags + "]\n"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (dateOffset ^ (dateOffset >>> 32)); + result = prime * result + ((file == null) ? 0 : file.hashCode()); + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final PdbFile other = (PdbFile) obj; + if (dateOffset != other.dateOffset) + return false; + if (file == null) { + if (other.file != null) + return false; + } else if (!file.equals(other.file)) + return false; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; + return true; + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java new file mode 100644 index 0000000..26ff5c4 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileByTimeAsc.java @@ -0,0 +1,14 @@ +package org.lucares.performance.db; + +import java.util.Comparator; + +public class PdbFileByTimeAsc implements Comparator { + + @Override + public int compare(final PdbFile o1, final PdbFile o2) { + + final long difference = o1.getDateOffset() - o2.getDateOffset(); + return difference < 0 ? -1 : (difference == 0 ? 0 : 1); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java new file mode 100644 index 0000000..ea9e917 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbFileIterator.java @@ -0,0 +1,109 @@ +package org.lucares.performance.db; + +import java.io.FileNotFoundException; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Iterator; +import java.util.Optional; +import java.util.Queue; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class PdbFileIterator implements Iterator, AutoCloseable { + + private final static Logger LOGGER = Logger.getLogger(PdbFileIterator.class.getCanonicalName()); + + private static final class EntrySupplier implements Supplier, AutoCloseable { + + private final Queue pdbFiles; + private PdbReader reader; + + public EntrySupplier(final Collection pdbFiles) { + super(); + this.pdbFiles = new ArrayDeque<>(pdbFiles); + } + + @Override + public Entry get() { + + if (reader == null) { + nextFile(); + } + final Optional optionalEntry = reader.readEntry(); + + return optionalEntry.orElseGet(() -> { + nextFile(); + if (reader == null) { + return null; + } else { + return reader.readEntry().orElse(null); + } + }); + + } + + private void nextFile() { + + if (reader != null) { + reader.close(); + reader = null; + } + + while (!pdbFiles.isEmpty()) { + final PdbFile pdbFile = pdbFiles.poll(); + try { + if (pdbFile.getFile().length() > 0) { + reader = new PdbReader(pdbFile); + break; + } else { + LOGGER.info("ignoring empty file " + pdbFile); + } + } catch (final FileNotFoundException e) { + LOGGER.log(Level.WARNING, "the pdbFile " + pdbFile.getFile() + " is missing", e); + } + } + } + + @Override + public void close() { + if (reader != null) { + reader.close(); + } + } + + } + + private final EntrySupplier supplier; + + private Optional next = Optional.empty(); + + public PdbFileIterator(final Collection pdbFiles) { + supplier = new EntrySupplier(pdbFiles); + } + + @Override + public boolean hasNext() { + final boolean result; + if (next.isPresent()) { + result = true; + } else { + next = Optional.ofNullable(supplier.get()); + result = next.isPresent(); + } + return result; + } + + @Override + public Entry next() { + final Entry result = next.orElseGet(supplier::get); + next = Optional.empty(); + return result; + } + + @Override + public void close() throws Exception { + supplier.close(); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java new file mode 100644 index 0000000..8d54057 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbReader.java @@ -0,0 +1,189 @@ +package org.lucares.performance.db; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Date; +import java.util.Optional; + +class PdbReader implements AutoCloseable { + + private static final int BYTES_PER_VALUE = 4; + private final RandomAccessFile data; + private final byte[] buffer = new byte[BYTES_PER_VALUE]; + private final PdbFile pdbFile; + + public PdbReader(final PdbFile pdbFile) throws FileNotFoundException { + super(); + this.pdbFile = pdbFile; + this.data = new RandomAccessFile(pdbFile.getFile(), "r"); + } + + /** + * Reads the next value. + *

+ * All values are non-negative. A negative return value indicates that the + * end of the file has been reached + * + * @return the value or -1 if end of stream has been reached + */ + public long readValue() { + assertPositionIsAValuePosition(); + + return read(); + } + + /** + * Reads the next date value. + * + * @return the date, or {@code null} if end of stream has been reached + * @throws IOException + */ + public Date readDate() { + assertPositionIsADatePosition(); + + final long value = read(); + if (value < 0) { + return null; + } + return new Date(pdbFile.getDateOffset() + value); + + } + + // visible for test + long read() { + try { + final int read = data.read(buffer); + + if (read < 0) { + return -1; + } + if (read != BYTES_PER_VALUE) { + throw new IllegalStateException("invalid file"); + } + return BitFiddling.makeLong(buffer[0], buffer[1], buffer[2], buffer[3]); + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + private void assertPositionIsADatePosition() { + try { + assertPositionIsValid(); + + if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 0) { + throw new IllegalStateException("file pointer is not at a date position: " + data.getFilePointer()); + } + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + private void assertPositionIsAValuePosition() { + assertPositionIsValid(); + try { + if ((data.getFilePointer() / BYTES_PER_VALUE) % 2 != 1) { + throw new IllegalStateException("file pointer is not at a value position: " + data.getFilePointer()); + } + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + private void assertPositionIsValid() { + try { + if (data.getFilePointer() % BYTES_PER_VALUE != 0) { + throw new IllegalStateException("file pointer is at an illegal position. It is at " + + data.getFilePointer() + " which is not divisible by " + BYTES_PER_VALUE); + } + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + /** + * Seek to the n-th value. + * + * @param n + */ + public void seek(final long n) { + try { + if (n < 0) { + throw new IllegalArgumentException("n must be non-negative, but was " + n); + } + + final long pos = n * BYTES_PER_VALUE; + + if (pos >= data.length()) { + throw new IllegalArgumentException("cannot seek to value " + n + ", because the file only has " + + (data.length() / BYTES_PER_VALUE) + " values"); + } + + data.seek(pos); + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + /** + * Seek to the end of the file. + *

+ * After this operation you can read {@code n} values. + * + * @param n + * seek to the n-th last value + * @throws ReadRuntimeException + * if the file does not have {@code n} entries + */ + public void seekTail(final long n) { + try { + if (n < 0) { + throw new IllegalArgumentException("n must be non-negative, but was " + n); + } + + final long pos = computeSeekPosition(n); + + data.seek(pos); + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + public boolean canSeekTail(final long n) { + try { + final long pos = computeSeekPosition(n); + return pos >= 0; + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + private long computeSeekPosition(final long n) throws IOException { + final long length = data.length(); + final long pos = length - BYTES_PER_VALUE * n; + return pos; + } + + @Override + public void close() { + try { + data.close(); + } catch (final IOException e) { + throw new ReadRuntimeException(e); + } + } + + public Optional readEntry() throws ReadRuntimeException { + final Date date = readDate(); + if (date == null) { + return Optional.empty(); + } + final long value = readValue(); + + if (value < 0) { + return Optional.empty(); + } + return Optional.of(new Entry(date, value)); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java new file mode 100644 index 0000000..055d551 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PdbWriter.java @@ -0,0 +1,67 @@ +package org.lucares.performance.db; + +import java.io.BufferedOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Date; + +class PdbWriter implements AutoCloseable { + + private static final long MAX_VALUE = 0xFF_FF_FF_FFL; + private static final boolean APPEND = true; + private final OutputStream outputStream; + private final PdbFile pdbFile; + + PdbWriter(final PdbFile pdbFile) throws FileNotFoundException { + this.pdbFile = pdbFile; + this.outputStream = new BufferedOutputStream(new FileOutputStream(pdbFile.getFile(), APPEND)); + } + + public PdbFile getFile() { + return pdbFile; + } + + public void write(final Entry entry) throws WriteException { + write(entry.getDate(), entry.getValue()); + } + + void write(final Date time, final long value) throws WriteException { + final long timeValue = time.getTime(); + final long adjustedValue = timeValue - pdbFile.getDateOffset(); + assertValueInRange(adjustedValue); + assertValueInRange(value); + + write(adjustedValue); + write(value); + } + + private void assertValueInRange(final long value) { + if (value < 0) { + throw new IllegalArgumentException("value must not be negative: " + value); + } + if (value > MAX_VALUE) { + throw new IllegalArgumentException("max value is " + MAX_VALUE + " value was: " + value); + } + } + + // visible for test + void write(final long value) throws WriteException { + assertValueInRange(value); + try { + outputStream.write(BitFiddling.long3(value)); + outputStream.write(BitFiddling.long2(value)); + outputStream.write(BitFiddling.long1(value)); + outputStream.write(BitFiddling.long0(value)); + } catch (final IOException e) { + throw new WriteException(e); + } + } + + @Override + public void close() throws IOException { + outputStream.flush(); + outputStream.close(); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java new file mode 100644 index 0000000..845fa9f --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/PerformanceDb.java @@ -0,0 +1,115 @@ +package org.lucares.performance.db; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import liquibase.exception.LiquibaseException; + +public class PerformanceDb implements AutoCloseable { + private static final Logger LOGGER = Logger.getLogger(PerformanceDb.class.getCanonicalName()); + + private final TagsToFile tagsToFile; + + public PerformanceDb(final Path dataDirectory) throws LiquibaseException { + tagsToFile = new TagsToFile(dataDirectory); + } + + public void put(final Date date, final long value, final Tags tags) throws WriteException { + put(new Entry(date, value), tags); + } + + public void put(final Entry entry, final Tags tags) throws WriteException { + put(Arrays.asList(entry), tags); + } + + public void put(final Iterable entries, final Tags tags) throws WriteException { + + final long start = System.nanoTime(); + double timeSpendInWrite = 0.0; + long count = 0; + PdbWriter writer = null; + PdbFile pdbFile = null; + try { + for (final Entry entry : entries) { + final Date date = entry.getDate(); + final long value = entry.getValue(); + + if (pdbFile == null // + || !pdbFile.getTimeRange().inRange(date)) // TODO @ahr + // correct + // would be + // to check + // if the + // date is + // in the + // available + // range + { + final long startWrite = System.nanoTime(); + pdbFile = tagsToFile.getFile(date, tags); + timeSpendInWrite += (System.nanoTime() - startWrite) / 1_000_000.0; + } + + if (writer == null || !writer.getFile().equals(pdbFile)) { + if (writer != null) { + writer.close(); + } + writer = new PdbWriter(pdbFile); + } + + writer.write(date, value); + count++; + } + + } catch (final IOException e) { + throw new WriteException(e); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (final IOException e) { + throw new WriteException(e); + } + } + + final double duration = (System.nanoTime() - start) / 1_000_000.0; + LOGGER.info("inserting " + count + " took " + duration + " ms of which " + timeSpendInWrite + + " were spend in write"); + } + } + + public List getAsList(final Tags tags) { + return get(tags).collect(Collectors.toList()); + } + + /** + * Return the entries as an unbound, ordered and non-parallel stream. + * + * @param tags + * @return {@link Stream} unbound, ordered and non-parallel + */ + public Stream get(final Tags tags) { + + final List pdbFiles = tagsToFile.getFilesMatchingTags(tags); + + final Iterator iterator = new PdbFileIterator(pdbFiles); + + final Spliterator spliterator = Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED); + return StreamSupport.stream(spliterator, false); + } + + @Override + public void close() throws Exception { + tagsToFile.close(); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Query.java b/performanceDb/src/main/java/org/lucares/performance/db/Query.java new file mode 100644 index 0000000..e099b4c --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/Query.java @@ -0,0 +1,18 @@ +package org.lucares.performance.db; + +final class Query { + static String createQuery(final Tags tags) { + final StringBuilder result = new StringBuilder(); + + for (final String key : tags.getKeys()) { + tags.getValue(key).ifPresent(value -> { + result.append(Fields.prefixedKey(key)); + result.append("="); + result.append(value); + result.append(" "); + }); + } + + return result.toString().trim(); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/ReadRuntimeException.java b/performanceDb/src/main/java/org/lucares/performance/db/ReadRuntimeException.java new file mode 100644 index 0000000..eb41def --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/ReadRuntimeException.java @@ -0,0 +1,18 @@ +package org.lucares.performance.db; + +public class ReadRuntimeException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public ReadRuntimeException(final String message, final Throwable cause) { + super(message, cause); + } + + public ReadRuntimeException(final String message) { + super(message); + } + + public ReadRuntimeException(final Throwable cause) { + super(cause); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java b/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java new file mode 100644 index 0000000..d117a8d --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/StorageUtils.java @@ -0,0 +1,11 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.nio.file.Path; + +public class StorageUtils { + + static File createStorageFile(final Path parent, final Day day, final String name) { + return new File(parent.toFile(), day.format(File.separator) + File.separator + name); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/Tags.java b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java new file mode 100644 index 0000000..97a5528 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/Tags.java @@ -0,0 +1,108 @@ +package org.lucares.performance.db; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.BiConsumer; + +public class Tags { + private final Map tags = new HashMap<>(); + + public Tags() { + super(); + } + + public Tags(final String key, final String value) { + super(); + tags.put(key, value); + } + + public Tags(final String key1, final String value1, final String key2, final String value2) { + super(); + tags.put(key1, value1); + tags.put(key2, value2); + } + + public Tags(final String key1, final String value1, final String key2, final String value2, final String key3, + final String value3) { + super(); + tags.put(key1, value1); + tags.put(key2, value2); + tags.put(key3, value3); + } + + public void put(final String key, final String value) { + tags.put(key, value); + } + + public Optional getValue(final String key) { + final String value = tags.get(key); + return Optional.ofNullable(value); + } + + public Set getKeys() { + return new TreeSet<>(tags.keySet()); + } + + public void forEach(final BiConsumer keyValueConsumer) { + for (final Map.Entry e : tags.entrySet()) { + keyValueConsumer.accept(e.getKey(), e.getValue()); + } + } + + @Override + public String toString() { + return String.valueOf(tags); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((tags == null) ? 0 : tags.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Tags other = (Tags) obj; + if (tags == null) { + if (other.tags != null) + return false; + } else if (!tags.equals(other.tags)) + return false; + return true; + } + + public String abbreviatedRepresentation() { + final StringBuilder result = new StringBuilder(); + final int maxLength = 200; + + final SortedSet keys = new TreeSet<>(tags.keySet()); + + final int cutAt = maxLength / (keys.size() * 2 + 2); + + for (final String key : keys) { + + result.append(substr(key, cutAt)); + result.append("-"); + result.append(substr(tags.get(key), cutAt)); + result.append("_"); + } + + return substr(result.toString(), maxLength); + } + + private static String substr(final String s, final int maxLength) { + return s.substring(0, Math.min(maxLength, s.length())); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java new file mode 100644 index 0000000..9edf53f --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/TagsToFile.java @@ -0,0 +1,191 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.lucares.ludb.Document; +import org.lucares.ludb.Field; +import org.lucares.ludb.FieldExistsException; +import org.lucares.ludb.FieldType; +import org.lucares.ludb.H2DB; + +import liquibase.exception.LiquibaseException; + +public class TagsToFile implements AutoCloseable, CollectionUtils { + + private final H2DB db; + private final Path dataDirectory; + + public TagsToFile(final Path dataDirectory) { + super(); + this.dataDirectory = dataDirectory; + try { + db = new H2DB(new File(dataDirectory.toFile(), "lu.db")); + try { + db.createField(Fields.DATE_OFFSET, FieldType.BIGINT); + } catch (final FieldExistsException e) { + // TODO @ahr ludb needs a hasField method, or a + // createIfNotExists + } + } catch (final LiquibaseException e) { + throw new RuntimeException(e); + } + } + + private List getFilesMatchingTagsExactly(final Tags tags) { + final List files = getFilesMatchingTags(tags); + + return filter(files, f -> f.getTags().equals(tags)); + } + + List getFilesMatchingTags(final Tags tags) { + final List result = new ArrayList<>(); + final String query = Query.createQuery(tags); + try { + final List searchResult = db.search(query); + + for (final Document document : searchResult) { + + final PdbFile pdbFile = toPdbFile(document); + + result.add(pdbFile); + } + } catch (final NullPointerException e) { + // TODO @ahr ludb should handle unknown fields better + } + + Collections.sort(result, new PdbFileByTimeAsc()); + return result; + } + + private PdbFile toPdbFile(final Document document) { + final File file = document.getFile(); + final long dateOffset = document.getPropertyLong(Fields.DATE_OFFSET); + final Tags tagsOfFile = toTags(document); + final PdbFile pdbFile = new PdbFile(dateOffset, file, tagsOfFile); + return pdbFile; + } + + private Tags toTags(final Document document) { + final Tags tagsOfFile = new Tags(); + + for (final String key : document.getProperties().keySet()) { + + if (Fields.isPrefixedKey(key)) { + final String value = document.getPropertyString(key); + tagsOfFile.put(Fields.stripPrefix(key), value); + } + } + return tagsOfFile; + } + + public PdbFile getFile(final Date date, final Tags tags) throws FileNotFoundException, IOException { + + final List pdbFiles = getFilesMatchingTagsExactly(tags); + final List preResult = new ArrayList<>(); + + PdbFile result; + for (final PdbFile pdbFile : pdbFiles) { + final boolean inRange = pdbFile.getTimeRange().inRange(date); + + if (inRange) { + final TimeRange availableTimeRange = getAvailableTimeRange(pdbFile); + + if (availableTimeRange.inRange(date)) { + preResult.add(pdbFile); + } + } + } + + if (preResult.isEmpty()) { + result = createNewPdbFile(date, tags); + } else { + if (preResult.size() != 1) { + throw new IllegalStateException("found not exactly one matching file: " + preResult); + } + result = preResult.get(0); + } + + return result; + } + + private TimeRange getAvailableTimeRange(final PdbFile pdbFile) throws FileNotFoundException, IOException { + + try (PdbReader reader = new PdbReader(pdbFile)) { + if (reader.canSeekTail(2)) { + reader.seekTail(2); + final Date lastDate = reader.readDate(); + + return new TimeRange(lastDate, pdbFile.getTimeRange().getTo()); + } else { + return pdbFile.getTimeRange(); + } + } + } + + private PdbFile createNewPdbFile(final Date date, final Tags tags) { + final File file; + final long dateOffset; + PdbFile result; + file = createNewFile(date, tags); + dateOffset = DateUtils.getDateOffset(date); + + db.addDocument(file); + + ensureFieldsExist(tags); + + tags.forEach((key, value) -> { + db.setProperty(file, Fields.prefixedKey(key), value); + }); + + db.setProperty(file, Fields.DATE_OFFSET, dateOffset); + + result = new PdbFile(dateOffset, file, tags); + return result; + } + + private void ensureFieldsExist(final Tags tags) { + final List fields = db.getAvailableFields(); + final Map fieldsMap = toMap(fields, Field::getName); + + tags.forEach((key, value) -> { + + final String prefixedKey = Fields.prefixedKey(key); + + final Field field = fieldsMap.get(prefixedKey); + if (field == null) { + db.createField(prefixedKey, FieldType.STRING); + } + }); + } + + private File createNewFile(final Date date, final Tags tags) { + final Day day = DateUtils.getDay(date); + final String name = tags.abbreviatedRepresentation() + UUID.randomUUID().toString(); + + final File result = StorageUtils.createStorageFile(dataDirectory, day, name); + try { + Files.createDirectories(result.getParentFile().toPath()); + Files.createFile(result.toPath()); + } catch (final IOException e) { + throw new IllegalStateException(e); // very unlikely + } + + return result; + } + + @Override + public void close() throws Exception { + db.close(); + } + +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java b/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java new file mode 100644 index 0000000..2ccd036 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/TimeRange.java @@ -0,0 +1,56 @@ +package org.lucares.performance.db; + +import java.util.Date; +import java.util.concurrent.TimeUnit; + +public class TimeRange { + private final Date from; + private final Date to; + + public TimeRange(final Date from, final Date to) { + if (!from.before(to)) { + throw new IllegalArgumentException("from date must be before to date. from: " + from + " to: " + to); + } + + this.from = from; + this.to = to; + } + + public Date getFrom() { + return from; + } + + public Date getTo() { + return to; + } + + public long length(final TimeUnit timeUnit) { + final long duration = to.getTime() - from.getTime(); + return timeUnit.convert(duration, TimeUnit.MILLISECONDS); + } + + public boolean inRange(final Date date) { + return from.compareTo(date) <= 0 && to.compareTo(date) >= 0; + } + + public boolean intersect(final TimeRange timeRange) { + return inRange(timeRange.from) // + || inRange(timeRange.to) // + || timeRange.inRange(from)// + || timeRange.inRange(to); + } + + @Override + public String toString() { + return "[" + from + ":" + to + "]"; + } + + public static TimeRange today() { + + final Date now = new Date(); + final Date from = DateUtils.getMidnightSameDay(now); + final Date to = DateUtils.getMidnightNextDay(now); + + return new TimeRange(from, to); + } +} diff --git a/performanceDb/src/main/java/org/lucares/performance/db/WriteException.java b/performanceDb/src/main/java/org/lucares/performance/db/WriteException.java new file mode 100644 index 0000000..471f2d0 --- /dev/null +++ b/performanceDb/src/main/java/org/lucares/performance/db/WriteException.java @@ -0,0 +1,15 @@ +package org.lucares.performance.db; + +public class WriteException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public WriteException(final String message, final Throwable cause) { + super(message, cause); + } + + public WriteException(final Throwable cause) { + super(cause); + } + +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java new file mode 100644 index 0000000..17820e2 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/PdbReaderWriterTest.java @@ -0,0 +1,115 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test +public class PdbReaderWriterTest { + + private Path dataDirectory; + + @BeforeMethod + public void beforeMethod() throws IOException { + dataDirectory = Files.createTempDirectory("pdb"); + } + + @AfterMethod + public void afterMethod() throws IOException { + FileUtils.delete(dataDirectory); + } + + @DataProvider(name = "providerWriteRead") + public Object[][] providerWriteRead() { + return new Object[][] { // + { 1 }, // + { 6 }, // + { 0xffffffffL },// + }; + } + + @Test(dataProvider = "providerWriteRead") + public void testWriteRead(final long value) throws Exception { + + final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); + final PdbFile pdbFile = PdbFile.today(file, new Tags()); + final Date now = new Date(); // TODO @ahr might fail at midnight + final Entry entry = new Entry(now, value); + + try (PdbWriter writer = new PdbWriter(pdbFile)) { + writer.write(entry); + } + + try (final PdbReader reader = new PdbReader(pdbFile)) { + final Entry actual = reader.readEntry().orElseThrow(() -> new AssertionError()); + + Assert.assertEquals(actual, entry); + } + } + + public void testSeekTail() throws Exception { + + final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); + final PdbFile pdbFile = PdbFile.today(file, new Tags()); + + try (PdbWriter writer = new PdbWriter(pdbFile)) { + writer.write(1); + writer.write(2); + writer.write(3); + writer.write(4); + writer.write(5); + } + + try (final PdbReader reader = new PdbReader(pdbFile)) { + reader.seekTail(2); + + final long four = reader.read(); + final long five = reader.read(); + + Assert.assertEquals(four, 4, "second last value"); + Assert.assertEquals(five, 5, "last value"); + + final long eof = reader.read(); + Assert.assertEquals(eof, -1, "end of file"); + } + } + + public void testSeek() throws Exception { + + final File file = Files.createTempFile(dataDirectory, "pdb", ".db").toFile(); + final PdbFile pdbFile = PdbFile.today(file, new Tags()); + + try (PdbWriter writer = new PdbWriter(pdbFile)) { + writer.write(1); + writer.write(2); + writer.write(3); + writer.write(4); + writer.write(5); + } + + try (final PdbReader reader = new PdbReader(pdbFile)) { + reader.seek(2); + + final long three = reader.read(); + final long four = reader.read(); + final long five = reader.read(); + + Assert.assertEquals(three, 3, "third value"); + Assert.assertEquals(four, 4, "fourth value"); + Assert.assertEquals(five, 5, "fifth value"); + + reader.seek(0); + final long first = reader.read(); + Assert.assertEquals(first, 1, "first value"); + + } + } +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java new file mode 100644 index 0000000..e3f33b0 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/PerformanceDbTest.java @@ -0,0 +1,162 @@ +package org.lucares.performance.db; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test +public class PerformanceDbTest { + + private Path dataDirectory; + + @BeforeMethod + public void beforeMethod() throws IOException { + dataDirectory = Files.createTempDirectory("pdb"); + } + + @AfterMethod + public void afterMethod() throws IOException { + FileUtils.delete(dataDirectory); + } + + public void testInsertRead() throws Exception { + + try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { + final Date date = new Date(); + final long value = 1; + final Tags tags = new Tags("myKey", "myValue"); + performanceDb.put(date, value, tags); + + final List stream = performanceDb.get(tags).collect(Collectors.toList()); + + Assert.assertEquals(stream.size(), 1); + + Assert.assertEquals(stream.get(0), new Entry(date, value)); + } + } + + public void testInsertIntoMultipleFilesRead() throws Exception { + + try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { + final Date dayOne = DateUtils.getDate(2016, 11, 1, 10, 0, 0); + final Date dayTwo = DateUtils.getDate(2016, 11, 2, 12, 34, 56); + final long valueOne = 1; + final long valueTwo = 2; + final Tags tags = new Tags("myKey", "myValue"); + + performanceDb.put(dayOne, valueOne, tags); + performanceDb.put(dayTwo, valueTwo, tags); + + final List stream = performanceDb.get(tags).collect(Collectors.toList()); + + Assert.assertEquals(stream.size(), 2); + + Assert.assertEquals(stream.get(0), new Entry(dayOne, valueOne)); + Assert.assertEquals(stream.get(1), new Entry(dayTwo, valueTwo)); + } + } + + private List generateEntries(final TimeRange timeRange, final long n) { + final List result = new ArrayList<>(); + final long differenceInMs = timeRange.length(TimeUnit.MILLISECONDS) / n; + long currentTime = timeRange.getFrom().getTime(); + + for (long i = 0; i < n; i++) { + final Date date = new Date(currentTime); + final long value = ThreadLocalRandom.current().nextInt(0, Integer.MAX_VALUE); + result.add(new Entry(date, value)); + + currentTime += differenceInMs; + } + return result; + } + + public void testAppendToExistingFile() throws Exception { + + try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { + + final TimeRange timeRange = TimeRange.today(); + final long numberOfEntries = 2; + + final Tags tags = new Tags("myKey", "one"); + final List entries = generateEntries(timeRange, numberOfEntries); + + printEntries(entries, ""); + + for (final Entry entry : entries) { + performanceDb.put(entry, tags); + } + + final List actualEntries = performanceDb.getAsList(tags); + Assert.assertEquals(actualEntries, entries); + + final File storageFileForToday = StorageUtils.createStorageFile(dataDirectory, + DateUtils.getDay(timeRange.getFrom()), "name doesn't matter"); + final File storageFolderForToday = storageFileForToday.getParentFile(); + final File[] filesInStorage = storageFolderForToday.listFiles(); + Assert.assertEquals(filesInStorage.length, 1, + "one file in storage, but was: " + Arrays.asList(filesInStorage)); + } + } + + public void testInsertIntoMultipleFilesWithDifferentTags() throws Exception { + + try (PerformanceDb performanceDb = new PerformanceDb(dataDirectory)) { + final Date from = DateUtils.getDate(2016, 1, 1, 00, 00, 00); + final Date to = DateUtils.getDate(2016, 12, 31, 23, 59, 59); + + final TimeRange timeRange = new TimeRange(from, to); + final long numberOfEntries = timeRange.length(TimeUnit.DAYS) * 2; // two + // entries + // per + // day + + final Tags tagsOne = new Tags("myKey", "one", "commonKey", "commonValue"); + final List entriesOne = generateEntries(timeRange, numberOfEntries); + printEntries(entriesOne, "one"); + performanceDb.put(entriesOne, tagsOne); + + final Tags tagsTwo = new Tags("myKey", "two", "commonKey", "commonValue"); + final List entriesTwo = generateEntries(timeRange, numberOfEntries); + printEntries(entriesTwo, "two"); + performanceDb.put(entriesTwo, tagsTwo); + + final Tags tagsThree = new Tags("myKey", "three", "commonKey", "commonValue"); + final List entriesThree = generateEntries(timeRange, numberOfEntries); + printEntries(entriesThree, "three"); + performanceDb.put(entriesThree, tagsThree); + + final List actualEntriesOne = performanceDb.getAsList(tagsOne); + Assert.assertEquals(actualEntriesOne, entriesOne); + + final List actualEntriesTwo = performanceDb.getAsList(tagsTwo); + Assert.assertEquals(actualEntriesTwo, entriesTwo); + + final List actualEntriesThree = performanceDb.getAsList(tagsThree); + Assert.assertEquals(actualEntriesThree, entriesThree); + + } + } + + private void printEntries(final List entriesOne, final String label) { + + int index = 0; + for (final Entry entry : entriesOne) { + System.out.printf("%4d %s %d (%s)\n", index, DateUtils.format(entry.getDate()), entry.getValue(), label); + index++; + } + } +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java new file mode 100644 index 0000000..0bf7f71 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/TagsToFilesTest.java @@ -0,0 +1,41 @@ +package org.lucares.performance.db; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; + +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@Test +public class TagsToFilesTest { + + private Path dataDirectory; + + @BeforeMethod + public void beforeMethod() throws IOException { + dataDirectory = Files.createTempDirectory("pdb"); + } + + @AfterMethod + public void afterMethod() throws IOException { + org.lucares.performance.db.FileUtils.delete(dataDirectory); + } + + public void test() throws Exception { + try (final TagsToFile tagsToFile = new TagsToFile(dataDirectory)) { + + final Date date = new Date(); + final Tags tags = new Tags("myKey", "myValue"); + + final PdbFile newFileForTags = tagsToFile.getFile(date, tags); + final PdbFile existingFileForTags = tagsToFile.getFile(date, tags); + + Assert.assertEquals(newFileForTags, existingFileForTags); + + } + } +} diff --git a/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java b/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java new file mode 100644 index 0000000..c252499 --- /dev/null +++ b/performanceDb/src/test/java/org/lucares/performance/db/TimeRangeTest.java @@ -0,0 +1,37 @@ +package org.lucares.performance.db; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@Test +public class TimeRangeTest { + + @DataProvider + Object[][] providerIntersect() { + final List result = new ArrayList<>(); + + final Date a = new Date(1000); + final Date b = new Date(2000); + final Date c = new Date(3000); + final Date d = new Date(4000); + + result.add(new Object[] { new TimeRange(a, b), new TimeRange(c, d), false }); + result.add(new Object[] { new TimeRange(a, c), new TimeRange(b, d), true }); + result.add(new Object[] { new TimeRange(a, d), new TimeRange(b, d), true }); + result.add(new Object[] { new TimeRange(a, d), new TimeRange(b, d), true }); + result.add(new Object[] { new TimeRange(a, b), new TimeRange(b, d), true }); + + return result.toArray(new Object[result.size()][]); + } + + @Test(dataProvider = "providerIntersect") + public void testIntersect(final TimeRange a, final TimeRange b, final boolean expected) throws Exception { + Assert.assertEquals(a.intersect(b), expected, a + " intersects " + b); + Assert.assertEquals(b.intersect(a), expected, a + " intersects " + b); + } +} diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..093c1ea --- /dev/null +++ b/settings.gradle @@ -0,0 +1,7 @@ +// include all projects with a build.gradle +// (this does not support nested projects) +File srcDir = new File(".") +FileCollection collection = files { srcDir.listFiles() } +collection.filter{ new File(it, "build.gradle").isFile() }.each{ include it.getName() } + +