buggy progress while filtering

raven/plugins/logfile/filterwidget.py

@@ -1,11 +1,13 @@
+import logging
 import os
 import re
 import tempfile
 import threading
+import time
 from typing import Optional, Callable
 
-from PySide6.QtCore import QRunnable, QThreadPool
+from PySide6.QtCore import QRunnable, QThreadPool, Signal, QThread, QObject
-from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QCheckBox, QPushButton
+from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QCheckBox, QPushButton, QLabel, QProgressBar
 
 from raven.ui.bigtext.bigtext import BigText
 from raven.ui.bigtext.logFileModel import LogFileModel
@@ -13,26 +15,28 @@ from raven.ui.bigtext.logFileModel import LogFileModel
 from raven.i18n import _
 from raven.pluginregistry import PluginRegistry
 
+log = logging.getLogger("filterwidget")
 
-class FilterTask(QRunnable):
+
+class FilterTask(QThread):
     aborted = False
 
+    on_before = Signal()
+    on_finish = Signal()
+    filter_progress = Signal(float)
+
     def __init__(
             self,
             source_model: LogFileModel,
             filter_model: LogFileModel,
             regex: re.Pattern,
             lock: threading.RLock,
-            filter_match_found_listeners: Callable[[int], None],
+            filter_match_found_listeners: Callable[[int], None]
-            on_before: Callable[[], None],
-            on_finish: Callable[[], None]
     ):
         super(FilterTask, self).__init__()
         self.source_model = source_model
         self.filter_model = filter_model
         self.regex = regex
-        self.on_before = on_before
-        self.on_finish = on_finish
         self.lock = lock
         self.filter_match_found_listeners = filter_match_found_listeners
 
@@ -41,21 +45,29 @@ class FilterTask(QRunnable):
 
         # the lock ensures that we only start a new search when the previous search already ended
         with self.lock:
-            # print("starting thread ", threading.currentThread())
+            print("starting thread ", threading.current_thread())
-            self.on_before()
+            self.on_before.emit()
 
             if self.aborted:
-                self.on_finish()
+                self.on_finish.emit()
 
             for listener in self.filter_match_found_listeners:
                 listener(-1, -1)  # notify listeners that a new search started
 
+            self.filter_progress.emit(0.0)
+
             try:
-                with open(self.source_model.get_file(), "rb") as source:
+                source_file = self.source_model.get_file()
+                file_size = os.stat(source_file).st_size
+                start = time.time()
+                with open(source_file, "rb") as source:
                     with open(self.filter_model.get_file(), "w+b") as target:
                         line_count = 0
                         lines_written = 0
+                        last_bytes_read = 0
+                        bytes_read = 0
                         while l := source.readline():
+                            bytes_read = bytes_read + len(l)
                             line_count = line_count + 1
                             line = l.decode("utf8", errors="ignore")
 
@@ -70,16 +82,28 @@ class FilterTask(QRunnable):
 
                             # sometime buffering can hide results for a while
                             # We force a flush periodically.
-                            if line_count % 10000 == 0 and lines_written > 0:
+                            if line_count % 10000 == 0:
+                                now = time.time()
+                                time_diff = (now - start)
+                                if time_diff > 0.5:
+                                    read_speed = ((bytes_read - last_bytes_read) / time_diff) / (1024 * 1024)
+                                    # todo progress disabled because of its detrimental effect on UI responsibility
+                                    # print("emit %f" % (bytes_read / file_size))
+                                    self.filter_progress.emit(bytes_read / file_size)
+                                    # self._progress_updater(bytes_read / file_size, read_speed)
+                                    last_bytes_read = bytes_read
+                                    start = time.time()
+
+                                if lines_written > 0:
                                     target.flush()
                                     lines_written = 0
 
                             if self.aborted:
-                                # print("aborted ", time.time())
+                                print("aborted ", time.time())
                                 break
             finally:
-                self.on_finish()
+                self.on_finish.emit()
-                # print("dome thread ", threading.currentThread())
+                print("dome thread ", threading.current_thread())
 
 
 class FilterWidget(QWidget):
@@ -98,6 +122,15 @@ class FilterWidget(QWidget):
         self.query_field = QLineEdit()
         self.query_field.textChanged.connect(self.filter_changed)
 
+        self._lbl_search_progress = QLabel("0%")
+        self._lbl_search_progress.setVisible(False)
+
+        self._progress_bar = QProgressBar();
+        self._progress_bar.setVisible(False)
+        self._progress_bar.setMinimum(0)
+        self._progress_bar.setMaximum(100)
+        self._progress_bar.setMaximumWidth(50)
+
         self.btn_cancel_search = QPushButton(_("Cancel"))
         self.btn_cancel_search.setVisible(False)
         self.btn_cancel_search.pressed.connect(self._cancel_search)
@@ -114,6 +147,8 @@ class FilterWidget(QWidget):
         filter_bar.layout = QHBoxLayout(filter_bar)
         filter_bar.layout.setContentsMargins(0, 0, 0, 0)
         filter_bar.layout.addWidget(self.query_field)
+        filter_bar.layout.addWidget(self._lbl_search_progress)
+        filter_bar.layout.addWidget(self._progress_bar)
         filter_bar.layout.addWidget(self.btn_cancel_search)
         filter_bar.layout.addWidget(self.ignore_case)
         filter_bar.layout.addWidget(self.is_regex)
@@ -185,7 +220,27 @@ class FilterWidget(QWidget):
             regex,
             self._lock,
             self.filter_match_found_listeners,
-            lambda: self.btn_cancel_search.setVisible(True),
-            lambda: self.btn_cancel_search.setVisible(False)
         )
-        QThreadPool.globalInstance().start(self.filter_task)
+        self.filter_task.on_before.connect(self._on_before)
+        self.filter_task.on_finish.connect(self._on_finish)
+        self.filter_task.filter_progress.connect(self._update_progress)
+        # self.filter_task.finished.connect(self.filter_task.deleteLater)
+        # super().connect(self.filter_task, FilterTask.filter_progress, self, self._update_progress)
+        # super().connect(self.filter_task, FilterTask.finished, self.filter_task, QObject.deleteLater)
+        self.filter_task.start()
+        # QThreadPool.globalInstance().start(self.filter_task)
+
+    def _on_before(self):
+        print("on_before")
+        self.btn_cancel_search.setVisible(True)
+        self._progress_bar.setVisible(True)
+
+    def _on_finish(self):
+        print("on_finish")
+        self.btn_cancel_search.setVisible(False)
+        self._progress_bar.setVisible(False)
+        self.filter_task.deleteLater()
+
+    def _update_progress(self, progress: float):
+        print("progress %f" % (progress))
+        self._progress_bar.setValue(progress * 100)