diff --git a/bigtext.py b/bigtext.py index c5cfedd..41ed807 100644 --- a/bigtext.py +++ b/bigtext.py @@ -2,7 +2,7 @@ import math import os import re import time -from typing import Optional, List +from typing import Optional, List, Callable import PyQt6.QtGui from PyQt6 import QtGui @@ -78,25 +78,36 @@ class BigText(QWidget): self.grid.setVerticalSpacing(0) self.setLayout(self.grid) - big_text = InnerBigText(self, model) - big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)) + self.big_text = InnerBigText(self, model) + self.big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)) self.h_scroll_bar = QScrollBar(Qt.Orientation.Horizontal) self.h_scroll_bar.setMinimum(0) self.h_scroll_bar.setMaximum(1) - self.h_scroll_bar.valueChanged.connect(big_text.h_scroll_event) + self.h_scroll_bar.valueChanged.connect(self.big_text.h_scroll_event) self.v_scroll_bar = ScaledScrollBar() # self.v_scroll_bar.setPageStep(1) - self.v_scroll_bar.valueChanged.connect(big_text.v_scroll_event) + self.v_scroll_bar.valueChanged.connect(self.big_text.v_scroll_event) - self.grid.addWidget(big_text, 0, 0) + self.grid.addWidget(self.big_text, 0, 0) self.grid.addWidget(self.h_scroll_bar, 1, 0) self.grid.addWidget(self.v_scroll_bar, 0, 1) def get_file(self): return self.model.get_file() + def add_line_click_listener(self, listener: Callable[[int], None]): + """ + + :param listener: a callable, the parameter is the byte offset of the clicked line + :return: + """ + self.big_text.line_click_listeners.append(listener) + + def scroll_to_byte(self, byte_offset: int): + self.big_text.scroll_to_byte(byte_offset) + def destruct(self): self.watchdog.destruct() pass @@ -123,6 +134,8 @@ class InnerBigText(QWidget): self._last_double_click_time = 0 self._last_double_click_line_number = -1 + self.line_click_listeners: [Callable[[int], None]] = [] + def keyPressEvent(self, e: QKeyEvent) -> None: # print("%s + %s" % (e.keyCombination().keyboardModifiers(), e.key())) @@ -187,6 +200,11 @@ class InnerBigText(QWidget): self.update() self.parent.v_scroll_bar.setValue(self._byte_offset) + def scroll_to_byte(self, byte_offset: int): + self._byte_offset = min(byte_offset, self.model.byte_count()) + self.update() + self.parent.v_scroll_bar.setValue(self._byte_offset) + def mousePressEvent(self, e: QtGui.QMouseEvent) -> None: if e.buttons() == Qt.MouseButton.LeftButton and e.modifiers() == Qt.KeyboardModifier.ShiftModifier: offset = self.to_byte_offset(e) @@ -210,6 +228,12 @@ class InnerBigText(QWidget): self.selection_highlight.set_end_byte(offset) self.update() + line_number = self.y_pos_to_line(e.pos().y()) + if line_number < len(self.lines): + line = self.lines[line_number] + for listener in self.line_click_listeners: + listener(line.byte_offset()) + def mouseDoubleClickEvent(self, e: QtGui.QMouseEvent) -> None: if e.buttons() == Qt.MouseButton.LeftButton: self._last_double_click_time = time.time() diff --git a/filterviewsyncer.py b/filterviewsyncer.py new file mode 100644 index 0000000..cc0910a --- /dev/null +++ b/filterviewsyncer.py @@ -0,0 +1,21 @@ +from bigtext import BigText + + +class FilterViewSyncer: + + def __init__(self, sync_view: BigText): + self._matches = {} + self._sync_view = sync_view + + def click_listener(self, byte_offset: int): + source_byte_offset = self._matches[byte_offset] if byte_offset in self._matches else None + # print("click %d -> %d (total hits %d)" % (byte_offset, source_byte_offset, len(self._matches))) + if source_byte_offset is not None: + self._sync_view.scroll_to_byte(source_byte_offset) + + def match_found(self, match_byte_offset: int, source_byte_offset: int): + # print("match %d" % match_byte_offset) + if match_byte_offset >= 0: + self._matches[match_byte_offset] = source_byte_offset + else: + self._matches = {} diff --git a/filterwidget.py b/filterwidget.py index 3df21c3..2ea16f0 100644 --- a/filterwidget.py +++ b/filterwidget.py @@ -22,6 +22,7 @@ class FilterTask(QRunnable): filter_model: LogFileModel, regex: re.Pattern, lock: threading.RLock, + filter_match_found_listeners: Callable[[int], None], on_before: Callable[[], None], on_finish: Callable[[], None] ): @@ -32,14 +33,19 @@ class FilterTask(QRunnable): self.on_before = on_before self.on_finish = on_finish self.lock = lock + self.filter_match_found_listeners = filter_match_found_listeners def run(self): # print("writing to tmp file", self.filter_model.get_file()) # the lock ensures that we only start a new search when the previous search already ended with self.lock: - #print("starting thread ", threading.currentThread()) + # print("starting thread ", threading.currentThread()) self.on_before() + + for listener in self.filter_match_found_listeners: + listener(-1, -1) # notify listeners that a new search started + try: with open(self.source_model.get_file(), "rb") as source: with open(self.filter_model.get_file(), "w+b") as target: @@ -50,9 +56,13 @@ class FilterTask(QRunnable): line = l.decode("utf8", errors="ignore") if self.regex.findall(line): - #time.sleep(0.5) - lines_written = lines_written +1 - target.write(line.encode("utf8")) + # time.sleep(0.5) + lines_written = lines_written + 1 + source_line_offset = source.tell() - len(l) + target_line_offset = target.tell() + for listener in self.filter_match_found_listeners: + listener(target_line_offset, source_line_offset) + target.write(l) # sometime buffering can hide results for a while # We force a flush periodically. @@ -112,6 +122,14 @@ class FilterWidget(QWidget): self.layout.addWidget(filter_bar) self.layout.addWidget(self.hits_view) + self.filter_match_found_listeners: [Callable[[int], None]] = [] + + def add_line_click_listener(self, listener: Callable[[int], None]): + self.hits_view.add_line_click_listener(listener) + + def add_filter_match_found_listener(self, listener: Callable[[int], None]): + self.filter_match_found_listeners.append(listener) + def destruct(self): # print("cleanup: ", self.tmpfilename) os.remove(self.tmpfilename) @@ -157,6 +175,7 @@ class FilterWidget(QWidget): self.filter_model, regex, self._lock, + self.filter_match_found_listeners, lambda: self.btn_cancel_search.setVisible(True), lambda: self.btn_cancel_search.setVisible(False) ) diff --git a/fulltabwidget.py b/fulltabwidget.py index c72982b..573032e 100644 --- a/fulltabwidget.py +++ b/fulltabwidget.py @@ -2,6 +2,7 @@ from PyQt6.QtWidgets import * from PyQt6.QtCore import * from bigtext import BigText +from filterviewsyncer import FilterViewSyncer from filterwidget import FilterWidget from logFileModel import LogFileModel @@ -13,6 +14,9 @@ class FullTabWidget(QWidget): self._model = model self.file_view = BigText(model) self.filter_hit_view = FilterWidget(self._model) + self.filter_view_syncer = FilterViewSyncer(self.file_view) + self.filter_hit_view.add_line_click_listener(self.filter_view_syncer.click_listener) + self.filter_hit_view.add_filter_match_found_listener(self.filter_view_syncer.match_found) self.layout = QVBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) diff --git a/int2intmaplike.py b/int2intmap.py similarity index 68% rename from int2intmaplike.py rename to int2intmap.py index 35b40e2..ab857d4 100644 --- a/int2intmaplike.py +++ b/int2intmap.py @@ -4,7 +4,7 @@ from logging import exception from typing import Optional -class Int2IntMapLike(): +class Int2IntMap(): """ A file used to map byte numbers of the filter view to byte numbers in the original file. Each line contains the two integers separated by a comma. @@ -25,16 +25,17 @@ class Int2IntMapLike(): def reset(self): self._handle.truncate(0) - def add(self, start: int, length: int, val: int): - line = "%d,%d,%d\n" % (start, length, val) + def add(self, key: int, val: int): + line = "%d,%d\n" % (key, val) length = len(line) offset = self._handle.tell() + len(self._buffer) if offset % self.blocksize + length > self.blocksize: # end of block: fill block fill_bytes = self.blocksize - offset % self.blocksize self._buffer = self._buffer + ("\n" * fill_bytes) - else: - self._buffer = self._buffer + line + self._buffer = self._buffer + line + if len(self._buffer) > self.blocksize * 100: + self._flush_buffer() def _flush_buffer(self): self._handle.write(self._buffer) @@ -48,9 +49,12 @@ class Int2IntMapLike(): if size == 0: return None total_blocks = math.ceil(size / self.blocksize) - step = math.ceil(total_blocks / 2) - offset = (step - 1) * self.blocksize - while step >= 1: + l = 0 + r = total_blocks - 1 + while r >= l: + mid = l + math.floor((r - l) / 2) + offset = mid * self.blocksize + self._handle.seek(offset) block = self._handle.read(self.blocksize) lines = block.split("\n") @@ -59,22 +63,23 @@ class Int2IntMapLike(): if len(line) == 0: continue token = line.split(",") - start = int(token[0]) - length = int(token[1]) - val = int(token[2]) + k = int(token[0]) + val = int(token[1]) - if key >= start and key - start < length: + if key == k: return val - tmp = key < start - if is_before != None and tmp != is_before: + tmp = key < k + if is_before is not None and tmp != is_before: return None - is_before = tmp + else: + is_before = tmp - if step == 1: - return None - - step = math.ceil(step / 2) if is_before: - offset = offset - step * self.blocksize + r = mid - 1 else: - offset = offset + step * self.blocksize + l = mid + 1 + return None + + def total_blocks(self) -> int: + size = os.stat(self._file).st_size + return math.ceil(size / self.blocksize) diff --git a/linetolinemap.py b/linetolinemap.py new file mode 100644 index 0000000..b9d8473 --- /dev/null +++ b/linetolinemap.py @@ -0,0 +1,22 @@ +import os +import tempfile +from typing import Optional + +from int2intmap import Int2IntMap + + +class LineToLineMap: + def __init__(self): + (handle, self.tmpfilename) = tempfile.mkstemp() + os.close(handle) + self._int2intmap = Int2IntMap(self.tmpfilename) + + def close(self): + self._int2intmap.close() + os.remove(self.tmpfilename) + + def add_line(self, key_byte_start: int, value_byte_start): + self._int2intmap.add(key_byte_start, value_byte_start) + + def get_line(self, key_byte_start: int) -> Optional[int]: + return self._int2intmap.find(key_byte_start) diff --git a/scribble.py b/scribble.py index a945520..cdfcd68 100644 --- a/scribble.py +++ b/scribble.py @@ -1,3 +1,4 @@ - +# extract icons from dll on windows +# https://mail.python.org/pipermail/python-win32/2009-April/009078.html print(min(2290538861, 2342622222)) diff --git a/testint2intmaplike.py b/testint2intmaplike.py index b5fcec7..fa7cfc5 100644 --- a/testint2intmaplike.py +++ b/testint2intmaplike.py @@ -2,15 +2,15 @@ import tempfile import unittest from os.path import join -from int2intmaplike import Int2IntMapLike +from int2intmap import Int2IntMap -class Int2IntMapLikeTest(unittest.TestCase): +class Int2IntMapLike(unittest.TestCase): def setUp(self): self.test_dir = tempfile.TemporaryDirectory() self.tmpfile = join(self.test_dir.name, "my.log") - self.map = Int2IntMapLike(self.tmpfile) + self.map = Int2IntMap(self.tmpfile) def tearDown(self): self.map.close() @@ -22,23 +22,24 @@ class Int2IntMapLikeTest(unittest.TestCase): def test_one_line_one_byte(self): map = self.map - map.add(10, 1, 1) # add only the key 10 + map.add(10, 1) # add the key 10 self.assertEqual(None, map.find(9)) # directly before self.assertEqual(1, map.find(10)) self.assertEqual(None, map.find(11)) # directly after def test_one_line_two_bytes(self): map = self.map - map.add(10, 2, 1) # added keys 10 and 11 + map.add(10, 1) # added key 10 + map.add(11, 2) # added key 11 self.assertEqual(None, map.find(9)) # directly before self.assertEqual(1, map.find(10)) - self.assertEqual(1, map.find(11)) + self.assertEqual(2, map.find(11)) self.assertEqual(None, map.find(12)) # directly after def test_two_lines(self): map = self.map - map.add(10, 1, 1) # added key 10 - map.add(12, 1, 2) # added key 12 + map.add(10, 1) # added key 10 + map.add(12, 2) # added key 12 self.assertEqual(None, map.find(9)) # directly before self.assertEqual(1, map.find(10)) self.assertEqual(None, map.find(11)) # between @@ -54,21 +55,21 @@ class Int2IntMapLikeTest(unittest.TestCase): # 20,5,2 # 30,5,3 # ... - for i in range(1, 20): - map.add(i * 10, 5, i) + # + # range(1,50) results in 6 blocks a 64 byte + for i in range(1, 50): + # print("add %d"%(i*10)) + map.add(i * 10, i) + # print("%d -> blocks: %d" %(i, map.total_blocks())) - self.assertEqual(2, map.find(20)) - self.assertEqual(7, map.find(71)) - self.assertEqual(13, map.find(134)) - self.assertEqual(19, map.find(194)) - - # values that are not in the map - self.assertEqual(None, map.find(0)) - self.assertEqual(None, map.find(9)) - self.assertEqual(None, map.find(15)) - self.assertEqual(None, map.find(16)) - self.assertEqual(None, map.find(107)) # a value in the second block - self.assertEqual(None, map.find(188)) # a value in the third block + for j in range(1, i * 10): + if j % 10 == 0: + # values that are in the map + # print("check %d" % (j * 10)) + self.assertEqual(j / 10, map.find(j)) + else: + # values that are not in the map + self.assertEqual(None, map.find(j)) if __name__ == '__main__':