ci: update ruff config and fix up

ruff.toml

@@ -1,4 +1,7 @@
-target-version = "py39"  # NOTE: inferred from pyproject.toml if present
+# NOTE: version is inferred from pyproject.toml if present
+#   however, if ruff.toml is separate, pyproject isn't even parsed..
+#   see https://github.com/astral-sh/ruff/issues/10299
+target-version = "py39"
 
 lint.extend-select = [
     "F",    # flakes rules -- default, but extend just in case
@@ -63,12 +66,6 @@ lint.ignore = [
 ## might be nice .. but later and I don't wanna make it strict
     "E402",  # Module level import not at top of file
 
-### maybe consider these soon
-    # sometimes it's useful to give a variable a name even if we don't use it as a documentation
-    # on the other hand, often is a sign of error
-    "F841",  # Local variable `count` is assigned to but never used
-###
-
     "RUF100",  # unused noqa -- handle later
     "RUF012",  # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs
 
@@ -109,7 +106,6 @@ lint.ignore = [
     "PLW0603",  # global variable update.. we usually know why we are doing this
     "PLW2901",  # for loop variable overwritten, usually this is intentional
 
-    "PT004",  # deprecated rule, will be removed later
     "PT011",  # pytest raises should is too broad
     "PT012",  # pytest raises should contain a single statement
 

src/promnesia/__main__.py

@@ -69,8 +69,7 @@ def iter_all_visits(sources_subset: Iterable[str | int] = ()) -> Iterator[Res[Db
             yield RuntimeError(f"Shouldn't have gotten this as a source: {source}")
             continue
 
-        # todo hmm it's not even used??
-        einfo = source.description
+        _einfo = source.description  # FIXME hmm it's not even used?? add as exception notes?
         for v in extract_visits(source, src=source.name):
             if hook is None:
                 yield v
@@ -409,7 +408,7 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
     # the only downside is storage. dunno.
     # worst case -- could use database?
 
-    with get_tmpdir() as tdir: # TODO??
+    with get_tmpdir() as _tdir:  # TODO what's the tmp dir for??
         if mode == 'index':
             errors = do_index(
                 config_file=args.config,

src/promnesia/cannon.py

@@ -122,8 +122,6 @@ def keep_query(self, q: str) -> int | None: # returns order
         qremove = default_qremove.union(self.qremove or {})
         # I suppose 'remove' is only useful for logging. we remove by default anyway
 
-        keep = False
-        remove = False
         qk = qkeep.get(q)
         if qk is not None:
             return qk
@@ -273,6 +271,9 @@ def _prenormalise(url: str) -> str:
     return url
 
 
+Left = Union[str, Sequence[str]]
+Right = tuple[str, str, str]
+
 def transform_split(split: SplitResult):
     netloc = canonify_domain(split.netloc)
 
@@ -282,10 +283,8 @@ def transform_split(split: SplitResult):
     fragment = split.fragment
 
     ID   = r'(?P<id>[^/]+)'
-    REST = r'(?P<rest>.*)'
+    # REST = r'(?P<rest>.*)'
 
-    Left = Union[str, Sequence[str]]
-    Right = tuple[str, str, str]
     # the idea is that we can unify certain URLs here and map them to the 'canonical' one
     # this is a dict only for grouping but should be a list really.. todo
     rules: dict[Left, Right] = {

src/promnesia/common.py

@@ -67,7 +67,7 @@ def file(cls, path: PathIsh, line: int | None=None, relative_to: Path | None=Non
             try:
                 # making it relative is a bit nicer for display
                 rel = rel.relative_to(relative_to)
-            except Exception as e:
+            except Exception:
                 pass # todo log/warn?
         loc = f'{rel}{lstr}'
         return cls.make(

src/promnesia/config.py

@@ -39,8 +39,6 @@ class Config(NamedTuple):
 
     @property
     def sources(self) -> Iterable[Res[Source]]:
-        idx = self.INDEXERS
-
         if len(self.INDEXERS) > 0:
             warnings.warn("'INDEXERS' is deprecated. Please use 'SOURCES'!", DeprecationWarning)
 

src/promnesia/server.py

@@ -363,7 +363,7 @@ def visited(request: VisitedRequest) -> VisitedResponse:
     logger = get_logger()
     logger.info('/visited %s %s', urls, client_version)
 
-    version = as_version(client_version)
+    _version = as_version(client_version)  # todo use it?
 
     nurls = [canonify(u) for u in urls]
     snurls = sorted(set(nurls))

src/promnesia/sources/plaintext.py

@@ -3,7 +3,7 @@
 from functools import lru_cache
 from pathlib import Path
 
-from promnesia.common import PathIsh, _is_windows, get_logger, get_tmpdir
+from promnesia.common import PathIsh, _is_windows
 
 # https://linux-and-mac-hacks.blogspot.co.uk/2013/04/use-grep-and-regular-expressions-to.html
 _URL_REGEX = r'\b(https?|ftp|file)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]*[-A-Za-z0-9+&@#/%=~_|]'
@@ -85,9 +85,6 @@ def _extract_from_file(path: str) -> Command:
 def extract_from_path(path: PathIsh) -> Command:
     pp = Path(path)
 
-    tdir = get_tmpdir()
-
-    logger = get_logger()
     if pp.is_dir(): # TODO handle archives here???
         return _extract_from_dir(str(pp))
 

src/promnesia/sources/pocket.py

@@ -16,7 +16,8 @@ def index() -> Results:
         loc = Loc.make(title=title, href=a.pocket_link)
         # Add a reverse locator so that the Promnesia browser extension shows a
         # link on the Pocket page back to the original URL.
-        loc_rev = Loc.make(title=title, href=a.url)
+        # FIXME need to actually use it
+        _loc_rev = Loc.make(title=title, href=a.url)
         hls = a.highlights
         excerpt = a.json.get('excerpt', None)
         if len(hls) == 0:

src/promnesia/sources/shellcmd.py

@@ -19,7 +19,6 @@
     _is_windows,
     extract_urls,
     file_mtime,
-    get_system_tz,
     now_tz,
 )
 
@@ -37,8 +36,6 @@ def index(command: str | Sequence[PathIsh]) -> Results:
         cmds = ' '.join(map(str, command))
         cmd = command
 
-    tz = get_system_tz()
-
     # ugh... on windows grep does something nasty? e.g:
     # grep --color=never -r -H -n -I -E http 'D:\\a\\promnesia\\promnesia\\tests\\testdata\\custom'
     # D:\a\promnesia\promnesia\tests\testdata\custom/file1.txt:1:Right, so this points at http://google.com

src/promnesia/sources/telegram_legacy.py

@@ -18,7 +18,6 @@
     echain,
     extract_urls,
     from_epoch,
-    get_logger,
 )
 
 from ..sqlite import sqlite_connection
@@ -39,8 +38,6 @@ def index(database: PathIsh, *, http_only: bool=False) -> Results:
     :param http_only:
         when true, do not collect IP-addresses and `python.py` strings
     """
-    logger = get_logger()
-
     path = Path(database)
     assert path.is_file(), path
 

src/promnesia/tests/common.py

@@ -59,6 +59,7 @@ def get_testdata(path: str) -> Path:
 @contextmanager
 def tmp_popen(*args, **kwargs):
     import psutil
+
     with psutil.Popen(*args, **kwargs) as p:
         try:
             yield p
@@ -99,6 +100,7 @@ def reset_filters():
 # TODO could be a TypeGuard from 3.10
 V = TypeVar('V')
 
+
 def unwrap(r: Res[V]) -> V:
     assert not isinstance(r, Exception), r
     return r

src/promnesia/tests/sources/test_hypothesis.py

@@ -12,7 +12,7 @@ def cfg() -> None:
         from promnesia.common import Source
         from promnesia.sources import hypothesis
 
-        SOURCES = [Source(hypothesis.index, name='hyp')]
+        SOURCES = [Source(hypothesis.index, name='hyp')]  # noqa: F841
 
     cfg_path = tmp_path / 'config.py'
     write_config(cfg_path, cfg)

src/promnesia/tests/sources/test_plaintext.py

@@ -5,13 +5,15 @@
 
 
 def test_plaintext_path_extractor() -> None:
-    visits = list(extract_visits(
-        Source(
-            shellcmd.index,
-            plaintext.extract_from_path(get_testdata('custom')),
-        ),
-        src='whatever',
-    ))
+    visits = list(
+        extract_visits(
+            Source(
+                shellcmd.index,
+                plaintext.extract_from_path(get_testdata('custom')),
+            ),
+            src='whatever',
+        )
+    )
     assert {unwrap(v).orig_url for v in visits} == {
         'http://google.com',
         'http://google.com/',

src/promnesia/tests/sources/test_shellcmd.py

@@ -9,13 +9,15 @@
 @pytest.mark.skipif(_is_windows, reason="no grep on windows")
 def test_via_grep() -> None:
 
-    visits = list(extract_visits(
-        Source(
-            shellcmd.index,
-            # meh. maybe should deprecate plain string here...
-            r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
-        ),
-        src='whatever',
-    ))
+    visits = list(
+        extract_visits(
+            Source(
+                shellcmd.index,
+                # meh. maybe should deprecate plain string here...
+                r"""grep -Eo -r --no-filename (http|https)://\S+ """ + str(get_testdata('custom')),
+            ),
+            src='whatever',
+        )
+    )
     # TODO I guess filtering of equivalent urls should rather be tested on something having context (e.g. org mode)
     assert len(visits) == 5

src/promnesia/tests/test_db_dump.py

@@ -158,6 +158,7 @@ def test_random_visit(visit: DbVisit) -> None:
 _dt_naive = datetime.fromisoformat('2023-11-14T23:11:01')
 _dt_aware = pytz.timezone('America/New_York').localize(_dt_naive)
 
+
 def make_testvisit(i: int) -> DbVisit:
     return DbVisit(
         norm_url=f'google.com/{i}',

src/promnesia/tests/test_extract.py

@@ -35,10 +35,12 @@ def test_urls_are_normalised() -> None:
     from ..sources import shellcmd
     from ..sources.plaintext import extract_from_path
 
-    visits = list(extract_visits(
-        source=Source(shellcmd.index, extract_from_path(get_testdata('normalise'))),
-        src='whatever',
-    ))
+    visits = list(
+        extract_visits(
+            source=Source(shellcmd.index, extract_from_path(get_testdata('normalise'))),
+            src='whatever',
+        )
+    )
     assert len(visits) == 7
 
     assert {unwrap(v).norm_url for v in visits} == {
@@ -59,6 +61,7 @@ def test_benchmark(count: int, gc_control) -> None:
         pytest.skip("test would be too slow on CI, only meant to run manually")
 
     from ..sources import demo
+
     source = Source(demo.index, count=count)
 
     total = ilen(extract_visits(source=source, src='whatever'))

src/promnesia/tests/test_indexer.py

@@ -32,7 +32,7 @@ def cfg1() -> None:
         from promnesia.common import Source
         from promnesia.sources import demo
 
-        SOURCES = [
+        SOURCES = [  # noqa: F841
             Source(demo.index, count=10, base_dt='2000-01-01', delta=30, name='demo1'),
             Source(demo.index, count=20, base_dt='2001-01-01', delta=30, name='demo2'),
         ]
@@ -48,7 +48,7 @@ def cfg2() -> None:
         from promnesia.common import Source
         from promnesia.sources import demo
 
-        SOURCES = [
+        SOURCES = [  # noqa: F841
             Source(demo.index, count=30, base_dt='2005-01-01', delta=30, name='demo2'),
             Source(demo.index, count=40, base_dt='2010-01-01', delta=30, name='demo3'),
         ]
@@ -73,7 +73,7 @@ def cfg_fast() -> None:
         from promnesia.common import Source
         from promnesia.sources import demo
 
-        SOURCES = [Source(demo.index, count=10)]
+        SOURCES = [Source(demo.index, count=10)]  # noqa: F841
 
     cfg_fast_path = tmp_path / 'cfg_fast.py'
     write_config(cfg_fast_path, cfg_fast)
@@ -82,7 +82,7 @@ def cfg_slow() -> None:
         from promnesia.common import Source
         from promnesia.sources import demo
 
-        SOURCES = [Source(demo.index, count=100_000)]
+        SOURCES = [Source(demo.index, count=100_000)]  # noqa: F841
 
     cfg_slow_path = tmp_path / 'cfg_slow.py'
     write_config(cfg_slow_path, cfg_slow)
@@ -122,11 +122,11 @@ def cfg(testdata, domain_to_filter) -> None:
         from promnesia.sources import shellcmd
         from promnesia.sources.plaintext import extract_from_path
 
-        FILTERS = [
+        FILTERS = [  # noqa: F841
             domain_to_filter,
         ]
 
-        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]
+        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]  # noqa: F841
 
     cfg_path = tmp_path / 'config.py'
     write_config(cfg_path, cfg, testdata=testdata, domain_to_filter=domain_to_filter)
@@ -147,7 +147,7 @@ def cfg(testdata: str) -> None:
         from promnesia.sources import shellcmd
         from promnesia.sources.plaintext import extract_from_path
 
-        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]
+        SOURCES = [Source(shellcmd.index, extract_from_path(testdata))]  # noqa: F841
 
     cfg_path = tmp_path / 'config.py'
     write_config(cfg_path, cfg, testdata=get_testdata('weird.txt'))
@@ -176,7 +176,7 @@ def indexer1():
         def indexer2():
             raise RuntimeError("in this case indexer itself crashed")
 
-        SOURCES = [Source(indexer1), Source(indexer2)]
+        SOURCES = [Source(indexer1), Source(indexer2)]  # noqa: F841
 
     cfg_path = tmp_path / 'config.py'
     write_config(cfg_path, cfg)
@@ -194,7 +194,7 @@ def cfg() -> None:
         from promnesia.common import Source
         from promnesia.sources import demo
 
-        SOURCES = [Source(demo.index, count=7, name='somename')]
+        SOURCES = [Source(demo.index, count=7, name='somename')]  # noqa: F841
 
         from collections.abc import Iterator
         from typing import cast