introduced a new custom file format used for backup and ingestion

The new file format reduces repetition, is easy to parse,
easy to generate in any language and is human readable.

pdb-ui/src/test/java/org/lucares/performance/db/ingestor/PdbTestUtil.java

@@ -4,8 +4,11 @@ import java.io.IOException;
 import java.net.ConnectException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.channels.SocketChannel;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -132,6 +135,24 @@ public class PdbTestUtil {
 		LOGGER.trace("closed sender connection");
 	}
 
+	public static void send(final Path file) throws IOException {
+		final SocketChannel outputChannel = connect();
+
+		try (final FileChannel inputChannel = FileChannel.open(file, StandardOpenOption.READ)) {
+			inputChannel.transferTo(0, Long.MAX_VALUE, outputChannel);
+		}
+
+		try {
+			// ugly workaround: the channel was closed too early and not all
+			// data was received
+			TimeUnit.MILLISECONDS.sleep(10);
+		} catch (final InterruptedException e) {
+			throw new IllegalStateException(e);
+		}
+		outputChannel.close();
+		LOGGER.trace("closed sender connection");
+	}
+
 	private static SocketChannel connect() throws IOException {
 
 		SocketChannel result = null;

pdb-ui/src/test/java/org/lucares/performance/db/ingestor/TcpIngestorTest.java

@@ -18,7 +18,9 @@ import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.lucares.collections.LongList;
+import org.lucares.pdb.datastore.internal.DataStore;
 import org.lucares.pdbui.TcpIngestor;
+import org.lucares.performance.db.PdbExport;
 import org.lucares.performance.db.PerformanceDb;
 import org.lucares.utils.file.FileUtils;
 import org.slf4j.Logger;
@@ -88,6 +90,63 @@ public class TcpIngestorTest {
 		}
 	}
 
+	public void testIngestDataViaTcpStream_CustomFormat() throws Exception {
+
+		final long dateA = Instant.now().toEpochMilli();
+		final long dateB = Instant.now().toEpochMilli() + 1;
+		final long dateC = Instant.now().toEpochMilli() - 1;
+		final String host = "someHost";
+
+		// 1. insert some data
+		try (TcpIngestor ingestor = new TcpIngestor(dataDirectory)) {
+
+			ingestor.start();
+
+			final long deltaEpochMilliB = dateB - dateA;
+			final long deltaEpochMilliC = dateC - dateB;
+
+			final String data = "#$0:host=someHost,pod=somePod\n"//
+					+ dateA + ",1,0\n"// previous date is 0, therefore the delta is dateA / using tags with id 0
+					+ "$1:host=someHost,pod=otherPod\n" //
+					+ deltaEpochMilliB + ",2,1\n" // dates are the delta the the previous date / using tags with id 1
+					+ deltaEpochMilliC + ",3,0"; // dates are the delta the the previous date / using tags with id 0
+
+			PdbTestUtil.send(data);
+		} catch (final Exception e) {
+			LOGGER.error("", e);
+			throw e;
+		}
+
+		// 2. export the data
+		final List<Path> exportFiles = PdbExport.export(dataDirectory, dataDirectory.resolve("export"));
+
+		// 3. delete database
+		FileUtils.delete(dataDirectory.resolve(DataStore.SUBDIR_STORAGE));
+
+		// 4. create a new database
+		try (TcpIngestor ingestor = new TcpIngestor(dataDirectory)) {
+			ingestor.start();
+			for (final Path exportFile : exportFiles) {
+				PdbTestUtil.send(exportFile);
+			}
+		}
+
+		// 5. check that the data is correctly inserted
+		try (PerformanceDb db = new PerformanceDb(dataDirectory)) {
+			final LongList result = db.get("host=" + host).singleGroup().flatMap();
+			Assert.assertEquals(result.size(), 6);
+
+			Assert.assertEquals(result.get(0), dateA);
+			Assert.assertEquals(result.get(1), 1);
+
+			Assert.assertEquals(result.get(2), dateC);
+			Assert.assertEquals(result.get(3), 3);
+
+			Assert.assertEquals(result.get(4), dateB);
+			Assert.assertEquals(result.get(5), 2);
+		}
+	}
+
 	@Test
 	public void testIngestionThreadDoesNotDieOnErrors() throws Exception {
 		final OffsetDateTime dateA = OffsetDateTime.ofInstant(Instant.ofEpochMilli(-1), ZoneOffset.UTC);