move files into a package structure

raven/i18n.py

@@ -3,7 +3,7 @@ import os
 from pathlib import Path
 
 from raven.pluginregistry import PluginRegistry
-from settingsstore import SettingsStore
+from raven.settings.settingsstore import SettingsStore
 
 settings = SettingsStore.load()
 locale = os.environ['LANG'] if os.environ['LANG'] else "en"

raven/mainwindow.py

@@ -5,16 +5,16 @@ from PySide6.QtWidgets import *
 from PySide6.QtGui import *
 from PySide6.QtCore import Qt
 
-import urlutils
-from cutesettings import CuteSettings
+from raven.util import urlutils
+from raven.settings.cutesettings import CuteSettings
 from raven.pluginregistry import PluginRegistry
 from raven.plugins.domain.menucontribution import MenuContribution, sort_menu_contributions
 from raven.plugins.domain.raction import RAction
 from raven.plugins.domain.rmenu import RMenu
-from settingsstore import SettingsStore
+from raven.settings.settingsstore import SettingsStore
 from highlightingdialog import HighlightingDialog
 from raven.ui.tabs import Tabs
-from urlutils import url_is_file
+from raven.util.urlutils import url_is_file
 from functools import reduce
 
 from raven.i18n import _

raven/plugins/logfileplugin.py

@@ -1,5 +1,5 @@
 import os.path
-from typing import Callable, Optional
+from typing import Optional
 
 from PySide6.QtWidgets import QMessageBox
 
@@ -7,7 +7,7 @@ from fulltabwidget import FullTabWidget
 from logFileModel import LogFileModel
 from raven.pluginbase import PluginBase
 from raven.plugins.ravenlog.Tab import Tab
-from settings import Settings
+from raven.settings.settings import Settings
 
 from raven.i18n import _
 

raven/plugins/openfileplugin.py

@@ -1,8 +1,6 @@
 import os
-from typing import Callable
 from pathlib import Path
 
-from PySide6.QtCore import QObject
 from PySide6.QtWidgets import QFileDialog
 
 from raven.pluginbase import PluginBase
@@ -10,7 +8,7 @@ from raven.pluginregistry import PluginRegistry
 from raven.plugins.domain.menucontribution import MenuContribution
 from raven.plugins.domain.raction import RAction
 from raven.plugins.domain.rmenu import RMenu
-from settings import Settings
+from raven.settings.settings import Settings
 
 from raven.i18n import _
 

raven/plugins/ravenlogplugin.py

@@ -13,7 +13,7 @@ from raven.plugins.domain.raction import RAction
 from raven.plugins.domain.rmenu import RMenu
 from raven.plugins.ravenlog.Tab import Tab
 from raven.i18n import _, locale
-from settings import Settings
+from raven.settings.settings import Settings
 
 
 class RavenLogPlugin(PluginBase):

raven/settings/cutesettings.py

@@ -0,0 +1,13 @@
+from PySide6.QtCore import QSettings
+
+
+class CuteSettings:
+
+    def __init__(self):
+        self._settings = QSettings("opentext", "ravenlog")
+
+    def set_value(self, key: str, value: any):
+        self._settings.setValue(key, value)
+
+    def value(self, key: str, default=None):
+        return self._settings.value(key, default)

raven/settings/settings.py

@@ -0,0 +1,23 @@
+from configparser import ConfigParser
+
+
+class Settings():
+
+    def __init__(self, session: ConfigParser):
+        self.session = session
+
+    def set_session(self, section: str, option: str, value: str):
+        return self.session.set(section, option, value)
+
+    def get_session(self, section: str, option: str) -> str:
+        return self.session.get(section, option)
+
+    def getint_session(self, section: str, option: str) -> int:
+        return self.session.getint(section, option)
+
+    def getboolean_session(self, section: str, option: str) -> bool:
+        return self.session.getboolean(section, option)
+
+    @staticmethod
+    def max_line_length():
+        return 4096

raven/settings/settingsstore.py

@@ -0,0 +1,45 @@
+import os
+import sys
+from pathlib import Path
+from os.path import join
+from configparser import ConfigParser
+
+from raven.settings.settings import Settings
+
+
+class SettingsStore():
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def _session_file() -> str:
+        if sys.platform == 'win32' or sys.platform == 'cygwin':
+            return join(Path.home(), "AppData", "Local", "ravenlog", "session.ini")
+        else:
+            return join(Path.home(), ".local", "share", "ravenlog", "session.ini")
+
+    @staticmethod
+    def load() -> Settings:
+        session = SettingsStore._load_session()
+        return Settings(session)
+
+    @staticmethod
+    def _load_session() -> ConfigParser:
+        session_file = SettingsStore._session_file()
+        session = ConfigParser()
+
+        # apply default settings
+        session.add_section('general')
+        session.set('general', 'font_size', '12')
+        session.set('general', 'highlight_search_term', 'True')
+        session.set('general', 'open_tab_on_save_as_file', 'True')
+        session.read(session_file)
+        return session
+
+    @staticmethod
+    def save(settings: Settings):
+        session_file = SettingsStore._session_file()
+        dir = os.path.dirname(session_file)
+        os.makedirs(dir, exist_ok=True)
+        with open(session_file, 'w+') as fp:
+            settings.session.write(fp)

raven/ui/tabs.py

@@ -4,7 +4,7 @@ from PySide6.QtWidgets import QWidget, QTabWidget, QVBoxLayout
 
 from raven.pluginregistry import PluginRegistry
 from raven.plugins.ravenlog.Tab import Tab
-from settings import Settings
+from raven.settings.settings import Settings
 
 
 class Tabs(QWidget):

raven/util/conversion.py

@@ -0,0 +1,29 @@
+import unittest
+
+def humanbytes(bytes: int) -> str:
+    """non-localized conversion of bytes to human readable strings"""
+    powers = {0: 'bytes', 1: 'KB', 2: 'MB', 3: 'GB', 4: 'TB', 5: 'PB', 6: 'EB'}
+    power = 1
+    result = "%d bytes" % bytes
+    while bytes >= 1024 ** power and power in powers:
+        result = "%.3f" % (bytes / (1024 ** power))
+        result = result.rstrip("0")
+        result = result.rstrip(".")
+        result = result + " " + powers[power]
+        power = power + 1
+    return result
+
+
+class TestLogFileModel(unittest.TestCase):
+
+    def test_humanbytes(self):
+        inputs = {
+            0: "0 bytes",
+            1023: "1023 bytes",
+            1024: "1 KB",
+            1048575: "1023.999 KB",
+            1048576: "1 MB",
+        }
+        for input in inputs.keys():
+            actual = humanbytes(input)
+            self.assertEqual(inputs[input], actual)

raven/util/int2intmap.py

@@ -0,0 +1,85 @@
+import math
+import os
+from logging import exception
+from typing import Optional
+
+
+class Int2IntMap():
+    """
+    A file used to map byte numbers of the filter view to byte numbers in the original file.
+    Each line contains the two integers separated by a comma.
+    The first column is sorted ascending. This allows us to do binary searches.
+    The file uses 4kb blocks. That means we add fill bytes (newlines) if a line would cross a 4kb block boundary.
+    """
+    blocksize = 4096
+
+    def __init__(self, file):
+        self._file = file
+        self._handle = open(file, "w+t")
+        self._buffer = ""
+
+    def close(self):
+        if not self._handle.closed:
+            self._handle.close()
+
+    def reset(self):
+        self._handle.truncate(0)
+
+    def add(self, key: int, val: int):
+        line = "%d,%d\n" % (key, val)
+        length = len(line)
+        offset = self._handle.tell() + len(self._buffer)
+        if offset % self.blocksize + length > self.blocksize:
+            # end of block: fill block
+            fill_bytes = self.blocksize - offset % self.blocksize
+            self._buffer = self._buffer + ("\n" * fill_bytes)
+        self._buffer = self._buffer + line
+        if len(self._buffer) > self.blocksize * 100:
+            self._flush_buffer()
+
+    def _flush_buffer(self):
+        self._handle.write(self._buffer)
+        self._buffer = ""
+        self._handle.flush()
+
+    def find(self, key: int) -> Optional[int]:
+        if (len(self._buffer)) > 0:
+            self._flush_buffer()
+        size = os.stat(self._file).st_size
+        if size == 0:
+            return None
+        total_blocks = math.ceil(size / self.blocksize)
+        l = 0
+        r = total_blocks - 1
+        while r >= l:
+            mid = l + math.floor((r - l) / 2)
+            offset = mid * self.blocksize
+
+            self._handle.seek(offset)
+            block = self._handle.read(self.blocksize)
+            lines = block.split("\n")
+            is_before = None
+            for line in lines:
+                if len(line) == 0:
+                    continue
+                token = line.split(",")
+                k = int(token[0])
+                val = int(token[1])
+
+                if key == k:
+                    return val
+                tmp = key < k
+                if is_before is not None and tmp != is_before:
+                    return None
+                else:
+                    is_before = tmp
+
+            if is_before:
+                r = mid - 1
+            else:
+                l = mid + 1
+        return None
+
+    def total_blocks(self) -> int:
+        size = os.stat(self._file).st_size
+        return math.ceil(size / self.blocksize)

raven/util/testint2intmaplike.py

@@ -0,0 +1,76 @@
+import tempfile
+import unittest
+from os.path import join
+
+from raven.util.int2intmap import Int2IntMap
+
+
+class Int2IntMapLike(unittest.TestCase):
+
+    def setUp(self):
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.tmpfile = join(self.test_dir.name, "my.log")
+        self.map = Int2IntMap(self.tmpfile)
+
+    def tearDown(self):
+        self.map.close()
+        self.test_dir.cleanup()
+
+    def test_empty_map(self):
+        map = self.map
+        self.assertEqual(None, map.find(0))
+
+    def test_one_line_one_byte(self):
+        map = self.map
+        map.add(10, 1)  # add the key 10
+        self.assertEqual(None, map.find(9))  # directly before
+        self.assertEqual(1, map.find(10))
+        self.assertEqual(None, map.find(11))  # directly after
+
+    def test_one_line_two_bytes(self):
+        map = self.map
+        map.add(10, 1)  # added key 10
+        map.add(11, 2)  # added key 11
+        self.assertEqual(None, map.find(9))  # directly before
+        self.assertEqual(1, map.find(10))
+        self.assertEqual(2, map.find(11))
+        self.assertEqual(None, map.find(12))  # directly after
+
+    def test_two_lines(self):
+        map = self.map
+        map.add(10, 1)  # added key 10
+        map.add(12, 2)  # added key 12
+        self.assertEqual(None, map.find(9))  # directly before
+        self.assertEqual(1, map.find(10))
+        self.assertEqual(None, map.find(11))  # between
+        self.assertEqual(2, map.find(12))
+        self.assertEqual(None, map.find(13))  # directly after
+
+    def test_fill_map(self):
+        map = self.map
+        map.blocksize = 64
+
+        # fill map with
+        # 10,5,1
+        # 20,5,2
+        # 30,5,3
+        # ...
+        #
+        # range(1,50) results in 6 blocks a 64 byte
+        for i in range(1, 50):
+            # print("add %d"%(i*10))
+            map.add(i * 10, i)
+            # print("%d -> blocks: %d" %(i, map.total_blocks()))
+
+            for j in range(1, i * 10):
+                if j % 10 == 0:
+                    # values that are in the map
+                    # print("check %d" % (j * 10))
+                    self.assertEqual(j / 10, map.find(j))
+                else:
+                    # values that are not in the map
+                    self.assertEqual(None, map.find(j))
+
+
+if __name__ == '__main__':
+    unittest.main()

raven/util/urlutils.py

@@ -0,0 +1,28 @@
+import os
+from urllib.parse import urlparse
+import sys
+
+
+def urls_to_path(urls: str) -> [str]:
+    result = []
+    url_list = urls.splitlines(keepends=False)
+    for url in url_list:
+        path = url_to_path(url)
+        result.append(path)
+    return result
+
+def url_to_path(url: str) -> str:
+    p = urlparse(url)
+    if sys.platform == 'win32' or sys.platform == 'cygwin':
+        return os.path.abspath(p.path[1:])
+    return os.path.abspath(os.path.join(p.netloc, p.path))
+
+
+def url_is_file(string: str) -> bool:
+    url_candidates = string.splitlines(keepends=False)
+    for url in url_candidates:
+        if url.startswith("file://"):
+            path = url_to_path(url)
+            if not os.path.isfile(path):
+                return False
+    return True