file drop support

- Add a folder where you can drop Zip files which will then be
  extracted on the fly and ingsted.
- CsvReaderSettings now contain TagMatcher that are applied to the
  first line and can be used to extract additional tags.
- Update to jdk 16 so that we can have records.
This commit is contained in:
2021-08-01 09:31:40 +02:00
parent 6d5cdbafca
commit 85ed5f1ccb
17 changed files with 430 additions and 14 deletions

View File

@@ -0,0 +1,61 @@
package org.lucares.pdbui;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Enumeration;
import java.util.Optional;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.lucares.pdb.datastore.Entries;
import org.lucares.pdb.datastore.RuntimeTimeoutException;
import org.lucares.performance.db.PerformanceDb;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Component
public class FileDropZipHandler implements FileDropFileTypeHandler {
private final PerformanceDb performanceDb;
private final FileDropConfigProvider configProvider;
@Autowired
public FileDropZipHandler(final PerformanceDb performanceDb, final FileDropConfigProvider configProvider) {
super();
this.performanceDb = performanceDb;
this.configProvider = configProvider;
}
@Override
public boolean isHandle(final Path file) {
return file.getFileName().toString().endsWith(".zip");
}
@Override
public void handle(final Path file) throws IOException, RuntimeTimeoutException, InterruptedException {
final ZipFile zipFile = new ZipFile(file.toFile());
final Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
final ZipEntry entry = entries.nextElement();
// System.out.println(entry.getName() + " isDir: " + entry.isDirectory());
if (entry.isDirectory()) {
continue;
}
final Optional<CsvReaderSettings> csvSettings = configProvider.provideCsvReaderSettings(entry.getName());
if (csvSettings.isPresent()) {
final ArrayBlockingQueue<Entries> queue = performanceDb.getQueue();
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, csvSettings.get());
try (final InputStream inputStream = new BufferedInputStream(zipFile.getInputStream(entry),
1024 * 1024)) {
csvToEntryTransformer.readCSV(inputStream);
}
}
}
}
}