try to fix ui freeze when filtering huge files

filterwidget.py

@@ -10,11 +10,10 @@ from PyQt6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QCheck
 
 from bigtext import BigText
 from logFileModel import LogFileModel
-import asyncio
+
 
 class FilterTask(QRunnable):
     aborted = False
-    future: asyncio.Future
 
     def __init__(
             self,
@@ -38,33 +37,46 @@ class FilterTask(QRunnable):
 
         # the lock ensures that we only start a new search when the previous search already ended
         with self.lock:
+            #print("starting thread ", threading.currentThread())
             self.on_before()
             try:
                 with open(self.source_model.get_file(), "rb") as source:
                     with open(self.filter_model.get_file(), "w+b") as target:
+                        line_count = 0
+                        lines_written = 0
                         while l := source.readline():
+                            line_count = line_count + 1
                             line = l.decode("utf8", errors="ignore")
 
                             if self.regex.findall(line):
+                                #time.sleep(0.5)
+                                lines_written = lines_written +1
                                 target.write(line.encode("utf8"))
 
+                            # sometime buffering can hide results for a while
+                            # We force a flush periodically.
+                            if line_count % 10000 == 0 and lines_written > 0:
+                                target.flush()
+                                lines_written = 0
+
                             if self.aborted:
-                                #print("aborted ", time.time())
+                                # print("aborted ", time.time())
                                 break
             finally:
                 self.on_finish()
+                #print("dome thread ", threading.currentThread())
 
 
 class FilterWidget(QWidget):
-    future = None
     filter_model: LogFileModel
     filter_task: Optional[FilterTask] = None
-    _lock = threading.RLock()
 
     def __init__(self, source_model: LogFileModel):
         super(FilterWidget, self).__init__()
         self.source_model = source_model
 
+        self._lock = threading.RLock()
+
         self.layout = QVBoxLayout(self)
         self.layout.setContentsMargins(0, 0, 0, 0)
 
@@ -99,18 +111,17 @@ class FilterWidget(QWidget):
         self.layout.addWidget(self.hits_view)
 
     def destruct(self):
-        #print("cleanup: ", self.tmpfilename)
+        # print("cleanup: ", self.tmpfilename)
         os.remove(self.tmpfilename)
 
     def _cancel_search(self):
         if self.filter_task:
-            #print("cancel started ", time.time())
+            # print("cancel started ", time.time())
             self.filter_task.aborted = True
 
     def filter_changed(self):
         query = self.query_field.text()
         ignore_case = self.ignore_case.isChecked()
-        is_regex = self.is_regex.isChecked()
         if len(query) == 0:
             return
 
@@ -119,7 +130,7 @@ class FilterWidget(QWidget):
 
         try:
             flags = re.IGNORECASE if ignore_case else 0
-            if is_regex:
+            if self.is_regex.isChecked():
                 regex = re.compile(query, flags=flags)
             else:
                 regex = re.compile(re.escape(query), flags=flags)
@@ -127,15 +138,12 @@ class FilterWidget(QWidget):
             # query was not a valid regex -> abort
             return
 
-
-
-
         self.filter_task = FilterTask(
             self.source_model,
             self.filter_model,
             regex,
             self._lock,
-            lambda : self.btn_cancel_search.setVisible(True),
-            lambda : self.btn_cancel_search.setVisible(False)
+            lambda: self.btn_cancel_search.setVisible(True),
+            lambda: self.btn_cancel_search.setVisible(False)
         )
         QThreadPool.globalInstance().start(self.filter_task)