From ce840819b246cedcc0b26539ada51beaa5c2a0e9 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 28 Oct 2021 08:45:24 +0200 Subject: [PATCH] try to fix ui freeze when filtering huge files --- bigtext.py | 10 ++++------ filterwidget.py | 36 ++++++++++++++++++++++-------------- logFileModel.py | 6 ++++-- main.py | 2 +- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/bigtext.py b/bigtext.py index a2bfb9a..8059708 100644 --- a/bigtext.py +++ b/bigtext.py @@ -42,15 +42,13 @@ class FileObserver(FileSystemEventHandler): class FileWatchdogThread(QRunnable): - observer = None - def __init__(self, big_text, file: str): super(FileWatchdogThread, self).__init__() self.file = file self.big_text = big_text + self.observer = Observer() def run(self) -> None: - self.observer = Observer() self.observer.schedule(FileObserver(self.big_text), self.file) self.observer.start() @@ -137,9 +135,9 @@ class InnerBigText(QWidget): self.scroll_by_lines(-lines_to_scroll) if e.key() == Qt.Key.Key_PageDown: self.scroll_by_lines(lines_to_scroll) - if e.key() == 16777235: # page up + if e.key() == 16777235: # page up self.scroll_by_lines(-3) - if e.key() == 16777237: # page down + if e.key() == 16777237: # page down self.scroll_by_lines(3) if e.modifiers() == Qt.KeyboardModifier.ControlModifier and e.key() == 67: # ctrl + c self.copy_selection() @@ -238,7 +236,7 @@ class InnerBigText(QWidget): cb.setText(selected_text) def paintEvent(self, event: QPaintEvent) -> None: - #print("paintEvent") + # print("paintEvent") painter = QPainter(self) painter.setFont(self.model.settings.font()) painter.setPen(QColor(0, 0, 0)) diff --git a/filterwidget.py b/filterwidget.py index 89b3b08..2fa2c2b 100644 --- a/filterwidget.py +++ b/filterwidget.py @@ -10,11 +10,10 @@ from PyQt6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QCheck from bigtext import BigText from logFileModel import LogFileModel -import asyncio + class FilterTask(QRunnable): aborted = False - future: asyncio.Future def __init__( self, @@ -38,33 +37,46 @@ class FilterTask(QRunnable): # the lock ensures that we only start a new search when the previous search already ended with self.lock: + #print("starting thread ", threading.currentThread()) self.on_before() try: with open(self.source_model.get_file(), "rb") as source: with open(self.filter_model.get_file(), "w+b") as target: + line_count = 0 + lines_written = 0 while l := source.readline(): + line_count = line_count + 1 line = l.decode("utf8", errors="ignore") if self.regex.findall(line): + #time.sleep(0.5) + lines_written = lines_written +1 target.write(line.encode("utf8")) + # sometime buffering can hide results for a while + # We force a flush periodically. + if line_count % 10000 == 0 and lines_written > 0: + target.flush() + lines_written = 0 + if self.aborted: - #print("aborted ", time.time()) + # print("aborted ", time.time()) break finally: self.on_finish() + #print("dome thread ", threading.currentThread()) class FilterWidget(QWidget): - future = None filter_model: LogFileModel filter_task: Optional[FilterTask] = None - _lock = threading.RLock() def __init__(self, source_model: LogFileModel): super(FilterWidget, self).__init__() self.source_model = source_model + self._lock = threading.RLock() + self.layout = QVBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) @@ -99,18 +111,17 @@ class FilterWidget(QWidget): self.layout.addWidget(self.hits_view) def destruct(self): - #print("cleanup: ", self.tmpfilename) + # print("cleanup: ", self.tmpfilename) os.remove(self.tmpfilename) def _cancel_search(self): if self.filter_task: - #print("cancel started ", time.time()) + # print("cancel started ", time.time()) self.filter_task.aborted = True def filter_changed(self): query = self.query_field.text() ignore_case = self.ignore_case.isChecked() - is_regex = self.is_regex.isChecked() if len(query) == 0: return @@ -119,7 +130,7 @@ class FilterWidget(QWidget): try: flags = re.IGNORECASE if ignore_case else 0 - if is_regex: + if self.is_regex.isChecked(): regex = re.compile(query, flags=flags) else: regex = re.compile(re.escape(query), flags=flags) @@ -127,15 +138,12 @@ class FilterWidget(QWidget): # query was not a valid regex -> abort return - - - self.filter_task = FilterTask( self.source_model, self.filter_model, regex, self._lock, - lambda : self.btn_cancel_search.setVisible(True), - lambda : self.btn_cancel_search.setVisible(False) + lambda: self.btn_cancel_search.setVisible(True), + lambda: self.btn_cancel_search.setVisible(False) ) QThreadPool.globalInstance().start(self.filter_task) diff --git a/logFileModel.py b/logFileModel.py index d0b72af..f8b26a8 100644 --- a/logFileModel.py +++ b/logFileModel.py @@ -1,5 +1,6 @@ import math import threading +import time from typing import List from line import Line import os @@ -7,11 +8,11 @@ from settings import Settings class LogFileModel: - _lock = threading.RLock() def __init__(self, file:str, settings: Settings): self.settings = settings self._file = os.path.realpath(file) + self._lock = threading.RLock() def get_file(self): return self._file @@ -36,11 +37,12 @@ class LogFileModel: #print("data(%s, %s, %s)" % (byte_offset, scroll_lines, lines)) lines_before_offset: List[Line] = [] lines_after_offset: List[Line] = [] - result: List[Line] = [] lines_to_find = lines + abs(scroll_lines) lines_to_return = math.ceil(lines) + #start = time.time() with self._lock: + #print("data lock acquision %.4f" % (time.time() -start)) # TODO handle lines longer than 4096 bytes # TODO abort file open after a few secons: https://docs.python.org/3/library/signal.html#example with open(self._file, 'rb') as f: diff --git a/main.py b/main.py index e1cd02b..0c07557 100644 --- a/main.py +++ b/main.py @@ -114,7 +114,7 @@ if __name__ == "__main__": # control back to python and allows python to react to signals timer = QTimer() timer.timeout.connect(lambda: None) - timer.start(500) + timer.start(100) window = MainWindow() window.show()