andi/perfdb
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Activity

add file drop handler

Browse Source 
You can define a folder and ingest files dropped into it.
This commit is contained in:
Andreas Huber
2021-08-07 13:31:44 +02:00
parent 85ed5f1ccb
commit 825bac24b9
10 changed files with 301 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
pdb-ui/src/main/java/org/lucares/pdbui/CsvReaderSettings.java
View File

@@ -2,6 +2,7 @@ package org.lucares.pdbui;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
@@ -12,8 +13,13 @@ import java.util.function.Function;
import org.lucares.pdbui.domain.TagMatcher;
import org.lucares.utils.Preconditions;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public final class CsvReaderSettings {
private final static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static String stripPrefixDefault(final String value) {
if (value.startsWith("Default")) {
return value.replaceFirst("Default", "");
@@ -169,7 +175,7 @@ public final class CsvReaderSettings {
private String comment = "#";
private List<TagMatcher> firstLineMatcher = new ArrayList<>();
private final List<TagMatcher> firstLineMatcher = new ArrayList<>();
public CsvReaderSettings() {
this("@timestamp", "duration", ",", new ColumnDefinitions());
@@ -244,7 +250,7 @@ public final class CsvReaderSettings {
}
public void putAdditionalTag(final Map<String, String> additionalTags) {
additionalTags.putAll(additionalTags);
this.additionalTags.putAll(additionalTags);
}
public Map<String, String> getAdditionalTags() {
@@ -267,8 +273,22 @@ public final class CsvReaderSettings {
return firstLineMatcher;
}
public void setFirstLineMatcher(final List<TagMatcher> firstLineMatcher) {
this.firstLineMatcher = firstLineMatcher;
public void setFirstLineMatcher(final Collection<TagMatcher> firstLineMatchers) {
this.firstLineMatcher.clear();
this.firstLineMatcher.addAll(firstLineMatchers);
}
public void addFirstLineMatcher(final TagMatcher tagMatcher) {
this.firstLineMatcher.add(tagMatcher);
}
public CsvReaderSettings copy() {
try {
final String json = OBJECT_MAPPER.writeValueAsString(this);
return OBJECT_MAPPER.readValue(json, CsvReaderSettings.class);
} catch (final JsonProcessingException e) {
throw new IllegalStateException(e);
}
}
@Override

3
pdb-ui/src/main/java/org/lucares/pdbui/CsvToEntryTransformer.java
View File

@@ -77,7 +77,8 @@ class CsvToEntryTransformer {
if (line[0] == comment) {
if (lineCounter == 1) {
final String lineAsString = new String(line, StandardCharsets.UTF_8);
final String lineAsString = new String(line, offsetInBuffer, length,
StandardCharsets.UTF_8);
final Tags firstLineTags = TagMatchExtractor.extractTags(lineAsString,
settings.getFirstLineMatcher());
additionalTags = additionalTags.add(firstLineTags);

15
pdb-ui/src/main/java/org/lucares/pdbui/FileDropConfigProvider.java
View File

@@ -5,8 +5,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.lucares.pdbui.domain.FileDropConfig;
import org.lucares.pdbui.domain.FileDropSettings;
@@ -52,22 +50,11 @@ public class FileDropConfigProvider {
final Map<String, String> variables = antPathMatcher.extractUriTemplateVariables(settings.match(),
file);
System.out.println("match found " + file + " regex: " + settings.match() + " " + variables);
final CsvReaderSettings csvSettings = settings.csvSettings();
final CsvReaderSettings csvSettings = settings.csvSettings().copy();
csvSettings.putAdditionalTag(variables);
return Optional.of(csvSettings);
}
}
return Optional.empty();
}
public static void main(final String[] args) {
final Matcher matcher = Pattern.compile("(?<source>.+)/(?<pod>.+)/(?<host>[^/]+)/performance.*.csv")
.matcher("web/vapsales01/0f5230761bb8a260e/performance.2020-10-05_000200_2.csv");
if (matcher.find()) {
System.out.println("match found");
} else {
System.out.println("not found");
}
}
}

27
pdb-ui/src/main/java/org/lucares/pdbui/FileDropHandler.java
View File

@@ -12,12 +12,13 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class FileDropHandler {
public class FileDropHandler implements AutoCloseable, DisposableBean {
private static final Logger LOGGER = LoggerFactory.getLogger(FileDropHandler.class);
@@ -40,8 +41,6 @@ public class FileDropHandler {
final WatchKey key = watchService.take();
final List<WatchEvent<?>> events = key.pollEvents();
for (final WatchEvent<?> watchEvent : events) {
System.out.printf("Event... kind=%s, count=%d, context=%s Context type=%s%n", watchEvent.kind(),
watchEvent.count(), watchEvent.context(), ((Path) watchEvent.context()).getClass());
final Path file = baseDir.resolve((Path) watchEvent.context());
for (final FileDropFileTypeHandler fileHandler : fileHandlers) {
@@ -53,15 +52,16 @@ public class FileDropHandler {
fileHandler.getClass(), e);
}
}
}
}
LOGGER.info("done ingesting file {}", file);
}
key.reset();
}
} catch (final InterruptedException e) {
try {
LOGGER.error("close watchService");
LOGGER.error("closing watchService in response to an interrupt");
watchService.close();
} catch (final IOException e1) {
LOGGER.error("failed to close watchService", e1);
@@ -72,6 +72,7 @@ public class FileDropHandler {
private final Path baseDir;
private final List<FileDropFileTypeHandler> fileHandlers;
private final FileWatchThread watchThread;
@Autowired
public FileDropHandler(@Value("${path.fileDrop}") final String baseDir,
@@ -84,8 +85,22 @@ public class FileDropHandler {
}
LOGGER.info("file drop location {}", this.baseDir);
final FileWatchThread watchThread = new FileWatchThread();
watchThread = new FileWatchThread();
watchThread.start();
}
public Path getBaseDir() {
return baseDir;
}
@Override
public void destroy() throws Exception {
close();
}
@Override
public void close() throws Exception {
watchThread.interrupt();
}
}

5
pdb-ui/src/main/java/org/lucares/pdbui/FileDropZipHandler.java
View File

@@ -50,7 +50,10 @@ public class FileDropZipHandler implements FileDropFileTypeHandler {
final Optional<CsvReaderSettings> csvSettings = configProvider.provideCsvReaderSettings(entry.getName());
if (csvSettings.isPresent()) {
final ArrayBlockingQueue<Entries> queue = performanceDb.getQueue();
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, csvSettings.get());
final CsvReaderSettings csvReaderSettings = csvSettings.get();
final CsvToEntryTransformer csvToEntryTransformer = new CsvToEntryTransformer(queue, csvReaderSettings);
try (final InputStream inputStream = new BufferedInputStream(zipFile.getInputStream(entry),
1024 * 1024)) {
csvToEntryTransformer.readCSV(inputStream);

11
pdb-ui/src/main/java/org/lucares/pdbui/domain/FileDropConfig.java
View File

@@ -1,19 +1,24 @@
package org.lucares.pdbui.domain;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
public class FileDropConfig {
private List<FileDropSettings> settings = new ArrayList<>();
private final List<FileDropSettings> settings = new ArrayList<>();
public List<FileDropSettings> getSettings() {
return settings;
}
public void setSettings(final List<FileDropSettings> settings) {
this.settings = settings;
public void setSettings(final Collection<FileDropSettings> settings) {
this.settings.addAll(settings);
}
public void addSettings(final FileDropSettings... dropSettings) {
this.settings.addAll(List.of(dropSettings));
}
@Override

5
pdb-ui/src/main/java/org/lucares/pdbui/domain/FileDropSettings.java
View File

@@ -2,6 +2,11 @@ package org.lucares.pdbui.domain;
import org.lucares.pdbui.CsvReaderSettings;
/**
* @param match ant style path matcher, e.g.
* {source}/{pod}/{host}/performance*.csv
* @param csvSettings
*/
public record FileDropSettings(String match, CsvReaderSettings csvSettings) {
}