try to fix ui freeze when filtering huge files

bigtext.py

@@ -42,15 +42,13 @@ class FileObserver(FileSystemEventHandler):
 
 class FileWatchdogThread(QRunnable):
 
-    observer = None
-
     def __init__(self, big_text, file: str):
         super(FileWatchdogThread, self).__init__()
         self.file = file
         self.big_text = big_text
+        self.observer = Observer()
 
     def run(self) -> None:
-        self.observer = Observer()
         self.observer.schedule(FileObserver(self.big_text), self.file)
         self.observer.start()
 
@@ -238,7 +236,7 @@ class InnerBigText(QWidget):
             cb.setText(selected_text)
 
     def paintEvent(self, event: QPaintEvent) -> None:
-        #print("paintEvent")
+        # print("paintEvent")
         painter = QPainter(self)
         painter.setFont(self.model.settings.font())
         painter.setPen(QColor(0, 0, 0))

filterwidget.py

@@ -10,11 +10,10 @@ from PyQt6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QCheck
 
 from bigtext import BigText
 from logFileModel import LogFileModel
-import asyncio
+
 
 class FilterTask(QRunnable):
     aborted = False
-    future: asyncio.Future
 
     def __init__(
             self,
@@ -38,33 +37,46 @@ class FilterTask(QRunnable):
 
         # the lock ensures that we only start a new search when the previous search already ended
         with self.lock:
+            #print("starting thread ", threading.currentThread())
             self.on_before()
             try:
                 with open(self.source_model.get_file(), "rb") as source:
                     with open(self.filter_model.get_file(), "w+b") as target:
+                        line_count = 0
+                        lines_written = 0
                         while l := source.readline():
+                            line_count = line_count + 1
                             line = l.decode("utf8", errors="ignore")
 
                             if self.regex.findall(line):
+                                #time.sleep(0.5)
+                                lines_written = lines_written +1
                                 target.write(line.encode("utf8"))
 
+                            # sometime buffering can hide results for a while
+                            # We force a flush periodically.
+                            if line_count % 10000 == 0 and lines_written > 0:
+                                target.flush()
+                                lines_written = 0
+
                             if self.aborted:
-                                #print("aborted ", time.time())
+                                # print("aborted ", time.time())
                                 break
             finally:
                 self.on_finish()
+                #print("dome thread ", threading.currentThread())
 
 
 class FilterWidget(QWidget):
-    future = None
     filter_model: LogFileModel
     filter_task: Optional[FilterTask] = None
-    _lock = threading.RLock()
 
     def __init__(self, source_model: LogFileModel):
         super(FilterWidget, self).__init__()
         self.source_model = source_model
 
+        self._lock = threading.RLock()
+
         self.layout = QVBoxLayout(self)
         self.layout.setContentsMargins(0, 0, 0, 0)
 
@@ -99,18 +111,17 @@ class FilterWidget(QWidget):
         self.layout.addWidget(self.hits_view)
 
     def destruct(self):
-        #print("cleanup: ", self.tmpfilename)
+        # print("cleanup: ", self.tmpfilename)
         os.remove(self.tmpfilename)
 
     def _cancel_search(self):
         if self.filter_task:
-            #print("cancel started ", time.time())
+            # print("cancel started ", time.time())
             self.filter_task.aborted = True
 
     def filter_changed(self):
         query = self.query_field.text()
         ignore_case = self.ignore_case.isChecked()
-        is_regex = self.is_regex.isChecked()
         if len(query) == 0:
             return
 
@@ -119,7 +130,7 @@ class FilterWidget(QWidget):
 
         try:
             flags = re.IGNORECASE if ignore_case else 0
-            if is_regex:
+            if self.is_regex.isChecked():
                 regex = re.compile(query, flags=flags)
             else:
                 regex = re.compile(re.escape(query), flags=flags)
@@ -127,15 +138,12 @@ class FilterWidget(QWidget):
             # query was not a valid regex -> abort
             return
 
-
-
-
         self.filter_task = FilterTask(
             self.source_model,
             self.filter_model,
             regex,
             self._lock,
-            lambda : self.btn_cancel_search.setVisible(True),
+            lambda: self.btn_cancel_search.setVisible(True),
-            lambda : self.btn_cancel_search.setVisible(False)
+            lambda: self.btn_cancel_search.setVisible(False)
         )
         QThreadPool.globalInstance().start(self.filter_task)

logFileModel.py

@@ -1,5 +1,6 @@
 import math
 import threading
+import time
 from typing import List
 from line import Line
 import os
@@ -7,11 +8,11 @@ from settings import Settings
 
 
 class LogFileModel:
-    _lock = threading.RLock()
 
     def __init__(self, file:str, settings: Settings):
         self.settings = settings
         self._file = os.path.realpath(file)
+        self._lock = threading.RLock()
 
     def get_file(self):
         return self._file
@@ -36,11 +37,12 @@ class LogFileModel:
         #print("data(%s, %s, %s)" % (byte_offset, scroll_lines, lines))
         lines_before_offset: List[Line] = []
         lines_after_offset: List[Line] = []
-        result: List[Line] = []
         lines_to_find = lines  + abs(scroll_lines)
         lines_to_return = math.ceil(lines)
 
+        #start = time.time()
         with self._lock:
+            #print("data lock acquision %.4f" % (time.time() -start))
             # TODO handle lines longer than 4096 bytes
             # TODO abort file open after a few secons: https://docs.python.org/3/library/signal.html#example
             with open(self._file, 'rb') as f:

main.py

@@ -114,7 +114,7 @@ if __name__ == "__main__":
     # control back to python and allows python to react to signals
     timer = QTimer()
     timer.timeout.connect(lambda: None)
-    timer.start(500)
+    timer.start(100)
 
     window = MainWindow()
     window.show()