andi/krowlog
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

link result view and source view

Browse Source
This commit is contained in:
Andreas Huber
2021-11-05 19:18:24 +01:00
parent e04c4a2ab7
commit f209156eea
8 changed files with 151 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
bigtext.py
View File

@@ -2,7 +2,7 @@ import math
import os
import re
import time
from typing import Optional, List
from typing import Optional, List, Callable
import PyQt6.QtGui
from PyQt6 import QtGui
@@ -78,25 +78,36 @@ class BigText(QWidget):
self.grid.setVerticalSpacing(0)
self.setLayout(self.grid)
big_text = InnerBigText(self, model)
big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding))
self.big_text = InnerBigText(self, model)
self.big_text.setSizePolicy(QSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding))
self.h_scroll_bar = QScrollBar(Qt.Orientation.Horizontal)
self.h_scroll_bar.setMinimum(0)
self.h_scroll_bar.setMaximum(1)
self.h_scroll_bar.valueChanged.connect(big_text.h_scroll_event)
self.h_scroll_bar.valueChanged.connect(self.big_text.h_scroll_event)
self.v_scroll_bar = ScaledScrollBar()
# self.v_scroll_bar.setPageStep(1)
self.v_scroll_bar.valueChanged.connect(big_text.v_scroll_event)
self.v_scroll_bar.valueChanged.connect(self.big_text.v_scroll_event)
self.grid.addWidget(big_text, 0, 0)
self.grid.addWidget(self.big_text, 0, 0)
self.grid.addWidget(self.h_scroll_bar, 1, 0)
self.grid.addWidget(self.v_scroll_bar, 0, 1)
def get_file(self):
return self.model.get_file()
def add_line_click_listener(self, listener: Callable[[int], None]):
"""
:param listener: a callable, the parameter is the byte offset of the clicked line
:return:
"""
self.big_text.line_click_listeners.append(listener)
def scroll_to_byte(self, byte_offset: int):
self.big_text.scroll_to_byte(byte_offset)
def destruct(self):
self.watchdog.destruct()
pass
@@ -123,6 +134,8 @@ class InnerBigText(QWidget):
self._last_double_click_time = 0
self._last_double_click_line_number = -1
self.line_click_listeners: [Callable[[int], None]] = []
def keyPressEvent(self, e: QKeyEvent) -> None:
# print("%s + %s" % (e.keyCombination().keyboardModifiers(), e.key()))
@@ -187,6 +200,11 @@ class InnerBigText(QWidget):
self.update()
self.parent.v_scroll_bar.setValue(self._byte_offset)
def scroll_to_byte(self, byte_offset: int):
self._byte_offset = min(byte_offset, self.model.byte_count())
self.update()
self.parent.v_scroll_bar.setValue(self._byte_offset)
def mousePressEvent(self, e: QtGui.QMouseEvent) -> None:
if e.buttons() == Qt.MouseButton.LeftButton and e.modifiers() == Qt.KeyboardModifier.ShiftModifier:
offset = self.to_byte_offset(e)
@@ -210,6 +228,12 @@ class InnerBigText(QWidget):
self.selection_highlight.set_end_byte(offset)
self.update()
line_number = self.y_pos_to_line(e.pos().y())
if line_number < len(self.lines):
line = self.lines[line_number]
for listener in self.line_click_listeners:
listener(line.byte_offset())
def mouseDoubleClickEvent(self, e: QtGui.QMouseEvent) -> None:
if e.buttons() == Qt.MouseButton.LeftButton:
self._last_double_click_time = time.time()

21
filterviewsyncer.py Normal file
View File

@@ -0,0 +1,21 @@
from bigtext import BigText
class FilterViewSyncer:
def __init__(self, sync_view: BigText):
self._matches = {}
self._sync_view = sync_view
def click_listener(self, byte_offset: int):
source_byte_offset = self._matches[byte_offset] if byte_offset in self._matches else None
# print("click %d -> %d (total hits %d)" % (byte_offset, source_byte_offset, len(self._matches)))
if source_byte_offset is not None:
self._sync_view.scroll_to_byte(source_byte_offset)
def match_found(self, match_byte_offset: int, source_byte_offset: int):
# print("match %d" % match_byte_offset)
if match_byte_offset >= 0:
self._matches[match_byte_offset] = source_byte_offset
else:
self._matches = {}

21
filterwidget.py
View File

@@ -22,6 +22,7 @@ class FilterTask(QRunnable):
filter_model: LogFileModel,
regex: re.Pattern,
lock: threading.RLock,
filter_match_found_listeners: Callable[[int], None],
on_before: Callable[[], None],
on_finish: Callable[[], None]
):
@@ -32,6 +33,7 @@ class FilterTask(QRunnable):
self.on_before = on_before
self.on_finish = on_finish
self.lock = lock
self.filter_match_found_listeners = filter_match_found_listeners
def run(self):
# print("writing to tmp file", self.filter_model.get_file())
@@ -40,6 +42,10 @@ class FilterTask(QRunnable):
with self.lock:
# print("starting thread ", threading.currentThread())
self.on_before()
for listener in self.filter_match_found_listeners:
listener(-1, -1) # notify listeners that a new search started
try:
with open(self.source_model.get_file(), "rb") as source:
with open(self.filter_model.get_file(), "w+b") as target:
@@ -52,7 +58,11 @@ class FilterTask(QRunnable):
if self.regex.findall(line):
# time.sleep(0.5)
lines_written = lines_written + 1
target.write(line.encode("utf8"))
source_line_offset = source.tell() - len(l)
target_line_offset = target.tell()
for listener in self.filter_match_found_listeners:
listener(target_line_offset, source_line_offset)
target.write(l)
# sometime buffering can hide results for a while
# We force a flush periodically.
@@ -112,6 +122,14 @@ class FilterWidget(QWidget):
self.layout.addWidget(filter_bar)
self.layout.addWidget(self.hits_view)
self.filter_match_found_listeners: [Callable[[int], None]] = []
def add_line_click_listener(self, listener: Callable[[int], None]):
self.hits_view.add_line_click_listener(listener)
def add_filter_match_found_listener(self, listener: Callable[[int], None]):
self.filter_match_found_listeners.append(listener)
def destruct(self):
# print("cleanup: ", self.tmpfilename)
os.remove(self.tmpfilename)
@@ -157,6 +175,7 @@ class FilterWidget(QWidget):
self.filter_model,
regex,
self._lock,
self.filter_match_found_listeners,
lambda: self.btn_cancel_search.setVisible(True),
lambda: self.btn_cancel_search.setVisible(False)
)

4
fulltabwidget.py
View File

@@ -2,6 +2,7 @@ from PyQt6.QtWidgets import *
from PyQt6.QtCore import *
from bigtext import BigText
from filterviewsyncer import FilterViewSyncer
from filterwidget import FilterWidget
from logFileModel import LogFileModel
@@ -13,6 +14,9 @@ class FullTabWidget(QWidget):
self._model = model
self.file_view = BigText(model)
self.filter_hit_view = FilterWidget(self._model)
self.filter_view_syncer = FilterViewSyncer(self.file_view)
self.filter_hit_view.add_line_click_listener(self.filter_view_syncer.click_listener)
self.filter_hit_view.add_filter_match_found_listener(self.filter_view_syncer.match_found)
self.layout = QVBoxLayout(self)
self.layout.setContentsMargins(0, 0, 0, 0)

43
int2intmaplike.py → int2intmap.py
View File

@@ -4,7 +4,7 @@ from logging import exception
from typing import Optional
class Int2IntMapLike():
class Int2IntMap():
"""
A file used to map byte numbers of the filter view to byte numbers in the original file.
Each line contains the two integers separated by a comma.
@@ -25,16 +25,17 @@ class Int2IntMapLike():
def reset(self):
self._handle.truncate(0)
def add(self, start: int, length: int, val: int):
line = "%d,%d,%d\n" % (start, length, val)
def add(self, key: int, val: int):
line = "%d,%d\n" % (key, val)
length = len(line)
offset = self._handle.tell() + len(self._buffer)
if offset % self.blocksize + length > self.blocksize:
# end of block: fill block
fill_bytes = self.blocksize - offset % self.blocksize
self._buffer = self._buffer + ("\n" * fill_bytes)
else:
self._buffer = self._buffer + line
if len(self._buffer) > self.blocksize * 100:
self._flush_buffer()
def _flush_buffer(self):
self._handle.write(self._buffer)
@@ -48,9 +49,12 @@ class Int2IntMapLike():
if size == 0:
return None
total_blocks = math.ceil(size / self.blocksize)
step = math.ceil(total_blocks / 2)
offset = (step - 1) * self.blocksize
while step >= 1:
l = 0
r = total_blocks - 1
while r >= l:
mid = l + math.floor((r - l) / 2)
offset = mid * self.blocksize
self._handle.seek(offset)
block = self._handle.read(self.blocksize)
lines = block.split("\n")
@@ -59,22 +63,23 @@ class Int2IntMapLike():
if len(line) == 0:
continue
token = line.split(",")
start = int(token[0])
length = int(token[1])
val = int(token[2])
k = int(token[0])
val = int(token[1])
if key >= start and key - start < length:
if key == k:
return val
tmp = key < start
if is_before != None and tmp != is_before:
tmp = key < k
if is_before is not None and tmp != is_before:
return None
else:
is_before = tmp
if step == 1:
if is_before:
r = mid - 1
else:
l = mid + 1
return None
step = math.ceil(step / 2)
if is_before:
offset = offset - step * self.blocksize
else:
offset = offset + step * self.blocksize
def total_blocks(self) -> int:
size = os.stat(self._file).st_size
return math.ceil(size / self.blocksize)

22
linetolinemap.py Normal file
View File

@@ -0,0 +1,22 @@
import os
import tempfile
from typing import Optional
from int2intmap import Int2IntMap
class LineToLineMap:
def __init__(self):
(handle, self.tmpfilename) = tempfile.mkstemp()
os.close(handle)
self._int2intmap = Int2IntMap(self.tmpfilename)
def close(self):
self._int2intmap.close()
os.remove(self.tmpfilename)
def add_line(self, key_byte_start: int, value_byte_start):
self._int2intmap.add(key_byte_start, value_byte_start)
def get_line(self, key_byte_start: int) -> Optional[int]:
return self._int2intmap.find(key_byte_start)

3
scribble.py
View File

@@ -1,3 +1,4 @@
# extract icons from dll on windows
# https://mail.python.org/pipermail/python-win32/2009-April/009078.html
print(min(2290538861, 2342622222))

43
testint2intmaplike.py
View File

@@ -2,15 +2,15 @@ import tempfile
import unittest
from os.path import join
from int2intmaplike import Int2IntMapLike
from int2intmap import Int2IntMap
class Int2IntMapLikeTest(unittest.TestCase):
class Int2IntMapLike(unittest.TestCase):
def setUp(self):
self.test_dir = tempfile.TemporaryDirectory()
self.tmpfile = join(self.test_dir.name, "my.log")
self.map = Int2IntMapLike(self.tmpfile)
self.map = Int2IntMap(self.tmpfile)
def tearDown(self):
self.map.close()
@@ -22,23 +22,24 @@ class Int2IntMapLikeTest(unittest.TestCase):
def test_one_line_one_byte(self):
map = self.map
map.add(10, 1, 1) # add only the key 10
map.add(10, 1) # add the key 10
self.assertEqual(None, map.find(9)) # directly before
self.assertEqual(1, map.find(10))
self.assertEqual(None, map.find(11)) # directly after
def test_one_line_two_bytes(self):
map = self.map
map.add(10, 2, 1) # added keys 10 and 11
map.add(10, 1) # added key 10
map.add(11, 2) # added key 11
self.assertEqual(None, map.find(9)) # directly before
self.assertEqual(1, map.find(10))
self.assertEqual(1, map.find(11))
self.assertEqual(2, map.find(11))
self.assertEqual(None, map.find(12)) # directly after
def test_two_lines(self):
map = self.map
map.add(10, 1, 1) # added key 10
map.add(12, 1, 2) # added key 12
map.add(10, 1) # added key 10
map.add(12, 2) # added key 12
self.assertEqual(None, map.find(9)) # directly before
self.assertEqual(1, map.find(10))
self.assertEqual(None, map.find(11)) # between
@@ -54,21 +55,21 @@ class Int2IntMapLikeTest(unittest.TestCase):
# 20,5,2
# 30,5,3
# ...
for i in range(1, 20):
map.add(i * 10, 5, i)
self.assertEqual(2, map.find(20))
self.assertEqual(7, map.find(71))
self.assertEqual(13, map.find(134))
self.assertEqual(19, map.find(194))
#
# range(1,50) results in 6 blocks a 64 byte
for i in range(1, 50):
# print("add %d"%(i*10))
map.add(i * 10, i)
# print("%d -> blocks: %d" %(i, map.total_blocks()))
for j in range(1, i * 10):
if j % 10 == 0:
# values that are in the map
# print("check %d" % (j * 10))
self.assertEqual(j / 10, map.find(j))
else:
# values that are not in the map
self.assertEqual(None, map.find(0))
self.assertEqual(None, map.find(9))
self.assertEqual(None, map.find(15))
self.assertEqual(None, map.find(16))
self.assertEqual(None, map.find(107)) # a value in the second block
self.assertEqual(None, map.find(188)) # a value in the third block
self.assertEqual(None, map.find(j))
if __name__ == '__main__':