link result view and source view

bigtext.py

@@ -2,7 +2,7 @@ import math
 import os
 import re
 import time
-from typing import Optional, List
+from typing import Optional, List, Callable
 import PyQt6.QtGui
 from PyQt6 import QtGui
 
@@ -78,25 +78,36 @@ class BigText(QWidget):
         self.grid.setVerticalSpacing(0)
         self.setLayout(self.grid)
 
-        big_text = InnerBigText(self, model)
+        self.big_text = InnerBigText(self, model)
-        big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding))
+        self.big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding))
 
         self.h_scroll_bar = QScrollBar(Qt.Orientation.Horizontal)
         self.h_scroll_bar.setMinimum(0)
         self.h_scroll_bar.setMaximum(1)
-        self.h_scroll_bar.valueChanged.connect(big_text.h_scroll_event)
+        self.h_scroll_bar.valueChanged.connect(self.big_text.h_scroll_event)
 
         self.v_scroll_bar = ScaledScrollBar()
         # self.v_scroll_bar.setPageStep(1)
-        self.v_scroll_bar.valueChanged.connect(big_text.v_scroll_event)
+        self.v_scroll_bar.valueChanged.connect(self.big_text.v_scroll_event)
 
-        self.grid.addWidget(big_text, 0, 0)
+        self.grid.addWidget(self.big_text, 0, 0)
         self.grid.addWidget(self.h_scroll_bar, 1, 0)
         self.grid.addWidget(self.v_scroll_bar, 0, 1)
 
     def get_file(self):
         return self.model.get_file()
 
+    def add_line_click_listener(self, listener: Callable[[int], None]):
+        """
+
+        :param listener: a callable, the parameter is the byte offset of the clicked line
+        :return:
+        """
+        self.big_text.line_click_listeners.append(listener)
+
+    def scroll_to_byte(self, byte_offset: int):
+        self.big_text.scroll_to_byte(byte_offset)
+
     def destruct(self):
         self.watchdog.destruct()
         pass
@@ -123,6 +134,8 @@ class InnerBigText(QWidget):
         self._last_double_click_time = 0
         self._last_double_click_line_number = -1
 
+        self.line_click_listeners: [Callable[[int], None]] = []
+
     def keyPressEvent(self, e: QKeyEvent) -> None:
 
         # print("%s + %s" % (e.keyCombination().keyboardModifiers(), e.key()))
@@ -187,6 +200,11 @@ class InnerBigText(QWidget):
         self.update()
         self.parent.v_scroll_bar.setValue(self._byte_offset)
 
+    def scroll_to_byte(self, byte_offset: int):
+        self._byte_offset = min(byte_offset, self.model.byte_count())
+        self.update()
+        self.parent.v_scroll_bar.setValue(self._byte_offset)
+
     def mousePressEvent(self, e: QtGui.QMouseEvent) -> None:
         if e.buttons() == Qt.MouseButton.LeftButton and e.modifiers() == Qt.KeyboardModifier.ShiftModifier:
             offset = self.to_byte_offset(e)
@@ -210,6 +228,12 @@ class InnerBigText(QWidget):
             self.selection_highlight.set_end_byte(offset)
             self.update()
 
+            line_number = self.y_pos_to_line(e.pos().y())
+            if line_number < len(self.lines):
+                line = self.lines[line_number]
+                for listener in self.line_click_listeners:
+                    listener(line.byte_offset())
+
     def mouseDoubleClickEvent(self, e: QtGui.QMouseEvent) -> None:
         if e.buttons() == Qt.MouseButton.LeftButton:
             self._last_double_click_time = time.time()

filterviewsyncer.py

@@ -0,0 +1,21 @@
+from bigtext import BigText
+
+
+class FilterViewSyncer:
+
+    def __init__(self, sync_view: BigText):
+        self._matches = {}
+        self._sync_view = sync_view
+
+    def click_listener(self, byte_offset: int):
+        source_byte_offset = self._matches[byte_offset] if byte_offset in self._matches else None
+        # print("click %d -> %d (total hits %d)" % (byte_offset, source_byte_offset, len(self._matches)))
+        if source_byte_offset is not None:
+            self._sync_view.scroll_to_byte(source_byte_offset)
+
+    def match_found(self, match_byte_offset: int, source_byte_offset: int):
+        # print("match %d" % match_byte_offset)
+        if match_byte_offset >= 0:
+            self._matches[match_byte_offset] = source_byte_offset
+        else:
+            self._matches = {}

filterwidget.py

@@ -22,6 +22,7 @@ class FilterTask(QRunnable):
             filter_model: LogFileModel,
             regex: re.Pattern,
             lock: threading.RLock,
+            filter_match_found_listeners: Callable[[int], None],
             on_before: Callable[[], None],
             on_finish: Callable[[], None]
     ):
@@ -32,14 +33,19 @@ class FilterTask(QRunnable):
         self.on_before = on_before
         self.on_finish = on_finish
         self.lock = lock
+        self.filter_match_found_listeners = filter_match_found_listeners
 
     def run(self):
         # print("writing to tmp file", self.filter_model.get_file())
 
         # the lock ensures that we only start a new search when the previous search already ended
         with self.lock:
-            #print("starting thread ", threading.currentThread())
+            # print("starting thread ", threading.currentThread())
             self.on_before()
+
+            for listener in self.filter_match_found_listeners:
+                listener(-1, -1)  # notify listeners that a new search started
+
             try:
                 with open(self.source_model.get_file(), "rb") as source:
                     with open(self.filter_model.get_file(), "w+b") as target:
@@ -50,9 +56,13 @@ class FilterTask(QRunnable):
                             line = l.decode("utf8", errors="ignore")
 
                             if self.regex.findall(line):
-                                #time.sleep(0.5)
+                                # time.sleep(0.5)
-                                lines_written = lines_written +1
+                                lines_written = lines_written + 1
-                                target.write(line.encode("utf8"))
+                                source_line_offset = source.tell() - len(l)
+                                target_line_offset = target.tell()
+                                for listener in self.filter_match_found_listeners:
+                                    listener(target_line_offset, source_line_offset)
+                                target.write(l)
 
                             # sometime buffering can hide results for a while
                             # We force a flush periodically.
@@ -112,6 +122,14 @@ class FilterWidget(QWidget):
         self.layout.addWidget(filter_bar)
         self.layout.addWidget(self.hits_view)
 
+        self.filter_match_found_listeners: [Callable[[int], None]] = []
+
+    def add_line_click_listener(self, listener: Callable[[int], None]):
+        self.hits_view.add_line_click_listener(listener)
+
+    def add_filter_match_found_listener(self, listener: Callable[[int], None]):
+        self.filter_match_found_listeners.append(listener)
+
     def destruct(self):
         # print("cleanup: ", self.tmpfilename)
         os.remove(self.tmpfilename)
@@ -157,6 +175,7 @@ class FilterWidget(QWidget):
             self.filter_model,
             regex,
             self._lock,
+            self.filter_match_found_listeners,
             lambda: self.btn_cancel_search.setVisible(True),
             lambda: self.btn_cancel_search.setVisible(False)
         )

fulltabwidget.py

@@ -2,6 +2,7 @@ from PyQt6.QtWidgets import *
 from PyQt6.QtCore import *
 
 from bigtext import BigText
+from filterviewsyncer import FilterViewSyncer
 from filterwidget import FilterWidget
 from logFileModel import LogFileModel
 
@@ -13,6 +14,9 @@ class FullTabWidget(QWidget):
         self._model = model
         self.file_view = BigText(model)
         self.filter_hit_view = FilterWidget(self._model)
+        self.filter_view_syncer = FilterViewSyncer(self.file_view)
+        self.filter_hit_view.add_line_click_listener(self.filter_view_syncer.click_listener)
+        self.filter_hit_view.add_filter_match_found_listener(self.filter_view_syncer.match_found)
 
         self.layout = QVBoxLayout(self)
         self.layout.setContentsMargins(0, 0, 0, 0)

int2intmap.py

@@ -4,7 +4,7 @@ from logging import exception
 from typing import Optional
 
 
-class Int2IntMapLike():
+class Int2IntMap():
     """
     A file used to map byte numbers of the filter view to byte numbers in the original file.
     Each line contains the two integers separated by a comma.
@@ -25,16 +25,17 @@ class Int2IntMapLike():
     def reset(self):
         self._handle.truncate(0)
 
-    def add(self, start: int, length: int, val: int):
+    def add(self, key: int, val: int):
-        line = "%d,%d,%d\n" % (start, length, val)
+        line = "%d,%d\n" % (key, val)
         length = len(line)
         offset = self._handle.tell() + len(self._buffer)
         if offset % self.blocksize + length > self.blocksize:
             # end of block: fill block
             fill_bytes = self.blocksize - offset % self.blocksize
             self._buffer = self._buffer + ("\n" * fill_bytes)
-        else:
         self._buffer = self._buffer + line
+        if len(self._buffer) > self.blocksize * 100:
+            self._flush_buffer()
 
     def _flush_buffer(self):
         self._handle.write(self._buffer)
@@ -48,9 +49,12 @@ class Int2IntMapLike():
         if size == 0:
             return None
         total_blocks = math.ceil(size / self.blocksize)
-        step = math.ceil(total_blocks / 2)
+        l = 0
-        offset = (step - 1) * self.blocksize
+        r = total_blocks - 1
-        while step >= 1:
+        while r >= l:
+            mid = l + math.floor((r - l) / 2)
+            offset = mid * self.blocksize
+
             self._handle.seek(offset)
             block = self._handle.read(self.blocksize)
             lines = block.split("\n")
@@ -59,22 +63,23 @@ class Int2IntMapLike():
                 if len(line) == 0:
                     continue
                 token = line.split(",")
-                start = int(token[0])
+                k = int(token[0])
-                length = int(token[1])
+                val = int(token[1])
-                val = int(token[2])
 
-                if key >= start and key - start < length:
+                if key == k:
                     return val
-                tmp = key < start
+                tmp = key < k
-                if is_before != None and tmp != is_before:
+                if is_before is not None and tmp != is_before:
                     return None
+                else:
                     is_before = tmp
 
-            if step == 1:
+            if is_before:
+                r = mid - 1
+            else:
+                l = mid + 1
         return None
 
-            step = math.ceil(step / 2)
+    def total_blocks(self) -> int:
-            if is_before:
+        size = os.stat(self._file).st_size
-                offset = offset - step * self.blocksize
+        return math.ceil(size / self.blocksize)
-            else:
-                offset = offset + step * self.blocksize

linetolinemap.py

@@ -0,0 +1,22 @@
+import os
+import tempfile
+from typing import Optional
+
+from int2intmap import Int2IntMap
+
+
+class LineToLineMap:
+    def __init__(self):
+        (handle, self.tmpfilename) = tempfile.mkstemp()
+        os.close(handle)
+        self._int2intmap = Int2IntMap(self.tmpfilename)
+
+    def close(self):
+        self._int2intmap.close()
+        os.remove(self.tmpfilename)
+
+    def add_line(self, key_byte_start: int, value_byte_start):
+        self._int2intmap.add(key_byte_start, value_byte_start)
+
+    def get_line(self, key_byte_start: int) -> Optional[int]:
+        return self._int2intmap.find(key_byte_start)

scribble.py

@@ -1,3 +1,4 @@
-
+# extract icons from dll on windows
+# https://mail.python.org/pipermail/python-win32/2009-April/009078.html
 
 print(min(2290538861, 2342622222))

testint2intmaplike.py

@@ -2,15 +2,15 @@ import tempfile
 import unittest
 from os.path import join
 
-from int2intmaplike import Int2IntMapLike
+from int2intmap import Int2IntMap
 
 
-class Int2IntMapLikeTest(unittest.TestCase):
+class Int2IntMapLike(unittest.TestCase):
 
     def setUp(self):
         self.test_dir = tempfile.TemporaryDirectory()
         self.tmpfile = join(self.test_dir.name, "my.log")
-        self.map = Int2IntMapLike(self.tmpfile)
+        self.map = Int2IntMap(self.tmpfile)
 
     def tearDown(self):
         self.map.close()
@@ -22,23 +22,24 @@ class Int2IntMapLikeTest(unittest.TestCase):
 
     def test_one_line_one_byte(self):
         map = self.map
-        map.add(10, 1, 1)  # add only the key 10
+        map.add(10, 1)  # add the key 10
         self.assertEqual(None, map.find(9))  # directly before
         self.assertEqual(1, map.find(10))
         self.assertEqual(None, map.find(11))  # directly after
 
     def test_one_line_two_bytes(self):
         map = self.map
-        map.add(10, 2, 1)  # added keys 10 and 11
+        map.add(10, 1)  # added key 10
+        map.add(11, 2)  # added key 11
         self.assertEqual(None, map.find(9))  # directly before
         self.assertEqual(1, map.find(10))
-        self.assertEqual(1, map.find(11))
+        self.assertEqual(2, map.find(11))
         self.assertEqual(None, map.find(12))  # directly after
 
     def test_two_lines(self):
         map = self.map
-        map.add(10, 1, 1)  # added key 10
+        map.add(10, 1)  # added key 10
-        map.add(12, 1, 2)  # added key 12
+        map.add(12, 2)  # added key 12
         self.assertEqual(None, map.find(9))  # directly before
         self.assertEqual(1, map.find(10))
         self.assertEqual(None, map.find(11))  # between
@@ -54,21 +55,21 @@ class Int2IntMapLikeTest(unittest.TestCase):
         # 20,5,2
         # 30,5,3
         # ...
-        for i in range(1, 20):
+        #
-            map.add(i * 10, 5, i)
+        # range(1,50) results in 6 blocks a 64 byte
-
+        for i in range(1, 50):
-        self.assertEqual(2, map.find(20))
+            # print("add %d"%(i*10))
-        self.assertEqual(7, map.find(71))
+            map.add(i * 10, i)
-        self.assertEqual(13, map.find(134))
+            # print("%d -> blocks: %d" %(i, map.total_blocks()))
-        self.assertEqual(19, map.find(194))
 
+            for j in range(1, i * 10):
+                if j % 10 == 0:
+                    # values that are in the map
+                    # print("check %d" % (j * 10))
+                    self.assertEqual(j / 10, map.find(j))
+                else:
                     # values that are not in the map
-        self.assertEqual(None, map.find(0))
+                    self.assertEqual(None, map.find(j))
-        self.assertEqual(None, map.find(9))
-        self.assertEqual(None, map.find(15))
-        self.assertEqual(None, map.find(16))
-        self.assertEqual(None, map.find(107))  # a value in the second block
-        self.assertEqual(None, map.find(188))  # a value in the third block
 
 
 if __name__ == '__main__':