Merge branch 'searxng:master' into master

.github/workflows/integration.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version:  ["3.8", "3.9", "3.10", "3.11"]
+        python-version:  ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
     - name: Checkout
       uses: actions/checkout@v4

.tool-versions

@@ -0,0 +1,2 @@
+python 3.12.0
+shellcheck 0.9.0

Dockerfile

@@ -15,7 +15,9 @@ ENV INSTANCE_NAME=searxng \
     MORTY_KEY= \
     MORTY_URL= \
     SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml \
-    UWSGI_SETTINGS_PATH=/etc/searxng/uwsgi.ini
+    UWSGI_SETTINGS_PATH=/etc/searxng/uwsgi.ini \
+    UWSGI_WORKERS=%k \
+    UWSGI_THREADS=4
 
 WORKDIR /usr/local/searxng
 

dockerfiles/uwsgi.ini

@@ -4,8 +4,12 @@ uid = searxng
 gid = searxng
 
 # Number of workers (usually CPU count)
-workers = %k
-threads = 4
+# default value: %k (= number of CPU core, see Dockerfile)
+workers = $(UWSGI_WORKERS)
+
+# Number of threads per worker
+# default value: 4 (see Dockerfile)
+threads = $(UWSGI_THREADS)
 
 # The right granted on the created socket
 chmod-socket = 666

docs/admin/index.rst

@@ -15,7 +15,7 @@ Administrator documentation
    installation-apache
    update-searxng
    answer-captcha
-   searx.botdetection
+   searx.limiter
    api
    architecture
    plugins

docs/admin/installation-docker.rst

@@ -92,6 +92,9 @@ instance using `docker run <https://docs.docker.com/engine/reference/run/>`_:
                 searxng/searxng
    2f998.... # container's ID
 
+The environment variables UWSGI_WORKERS and UWSGI_THREADS overwrite the default
+number of UWSGI processes and UWSGI threads specified in `/etc/searxng/uwsgi.ini`.
+
 Open your WEB browser and visit the URL:
 
 .. code:: sh

docs/admin/searx.limiter.rst

@@ -0,0 +1,17 @@
+.. _limiter:
+
+=======
+Limiter
+=======
+
+.. sidebar:: info
+
+   The limiter requires a :ref:`Redis <settings redis>` database.
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+.. automodule:: searx.limiter
+   :members:

docs/admin/settings/settings_server.rst

@@ -12,6 +12,7 @@
        bind_address: "127.0.0.1"
        secret_key: "ultrasecretkey"           # change this!
        limiter: false
+       public_instance: false
        image_proxy: false
        default_http_headers:
          X-Content-Type-Options : nosniff
@@ -20,7 +21,6 @@
          X-Robots-Tag : noindex, nofollow
          Referrer-Policy : no-referrer
 
-
 ``base_url`` : ``$SEARXNG_URL`` :ref:`buildenv <make buildenv>`
   The base URL where SearXNG is deployed.  Used to create correct inbound links.
   If you change the value, don't forget to rebuild instance's environment
@@ -36,11 +36,19 @@
 ``secret_key`` : ``$SEARXNG_SECRET``
   Used for cryptography purpose.
 
-.. _limiter:
-
 ``limiter`` :
   Rate limit the number of request on the instance, block some bots.  The
-  :ref:`limiter src` requires a :ref:`settings redis` database.
+  :ref:`limiter` requires a :ref:`settings redis` database.
+
+.. _public_instance:
+
+``public_instance`` :
+
+  Setting that allows to enable features specifically for public instances (not
+  needed for local usage).  By set to ``true`` the following features are
+  activated:
+
+  - :py:obj:`searx.botdetection.link_token` in the :ref:`limiter`
 
 .. _image_proxy:
 

docs/conf.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 import  sys, os
+from pathlib import Path
 from pallets_sphinx_themes import ProjectLink
 
 from searx import get_setting
@@ -13,7 +14,6 @@
 copyright = 'SearXNG team'
 author = 'SearXNG team'
 release, version = VERSION_STRING, VERSION_STRING
-
 SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng'
 ISSUE_URL = get_setting('brand.issue_url')
 DOCS_URL = get_setting('brand.docs_url')
@@ -22,6 +22,9 @@
 CONTACT_URL = get_setting('general.contact_url')
 WIKI_URL = get_setting('brand.wiki_url')
 
+SOURCEDIR = Path(__file__).parent.parent / "searx"
+os.environ['SOURCEDIR'] = str(SOURCEDIR)
+
 # hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set
 #       to string 'none' [2]
 #

docs/dev/index.rst

@@ -6,6 +6,7 @@ Developer documentation
    :maxdepth: 2
 
    quickstart
+   rtm_asdf
    contribution_guide
    engines/index
    search_api

docs/dev/rtm_asdf.rst

@@ -0,0 +1,121 @@
+==================
+Runtime Management
+==================
+
+The runtimes are managed with asdf and are activated in this project via the
+`.tool-versions <.tool-versions>`_. If you have not yet installed asdf_, then
+chapter :ref:`introduce asdf` may be of help to you.
+
+.. contents::
+   :depth: 2
+   :local:
+   :backlinks: entry
+
+
+Get started
+===========
+
+If you have asdf installed you can install the runtimes of this project by:
+
+.. code:: bash
+
+   $ cd /path/to/searxng
+   $ asdf install          # will install runtimes listed in .tool-versions
+   ...
+
+Manage Versions
+===============
+
+If you want to perform a ``test`` with special runtime versions of nodejs,
+python or shellcheck, you can patch the ``.tool-versions``:
+
+.. code:: diff
+
+   --- a/.tool-versions
+   +++ b/.tool-versions
+   @@ -1,2 +1,2 @@
+   -python 3.12.0
+   -shellcheck 0.9.0
+   +python 3.11.6
+   +shellcheck 0.8.0
+
+To install use ``asdf install`` again.  If the runtime tools have changed, any
+existing (nodejs and python) environments should be cleaned up with a ``make
+clean``.
+
+.. code:: bash
+
+   $ asdf install
+   ...
+   $ make clean test
+
+
+.. _introduce asdf:
+
+Introduce asdf
+==============
+
+To `download asdf`_ and `install asdf`_:
+
+.. code:: bash
+
+   $ git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch <version>
+   $ echo '. "$HOME/.asdf/asdf.sh"' >> ~/.bashrc
+   $ echo '. "$HOME/.asdf/completions/asdf.bash"' >> ~/.bashrc
+
+Start a new shell and try to `install plugins`_:
+
+.. code:: bash
+
+   $ asdf plugin-list-all | grep -E '(golang|python|nodejs|shellcheck).git'
+   golang                        https://github.com/asdf-community/asdf-golang.git
+   nodejs                        https://github.com/asdf-vm/asdf-nodejs.git
+   python                        https://github.com/danhper/asdf-python.git
+   shellcheck                    https://github.com/luizm/asdf-shellcheck.git
+
+   $ asdf plugin add golang https://github.com/asdf-community/asdf-golang.git
+   $ asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git
+   $ asdf plugin add python https://github.com/danhper/asdf-python.git
+   $ asdf plugin add shellcheck https://github.com/luizm/asdf-shellcheck.git
+
+Each plugin has dependencies, to compile runtimes visit the URLs from above and
+look out for the dependencies you need to install on your OS, on Debian for the
+runtimes listed above you will need:
+
+.. code:: bash
+
+  $ sudo apt update
+  $ sudo apt install \
+         dirmngr gpg curl gawk coreutils build-essential libssl-dev zlib1g-dev \
+         libbz2-dev libreadline-dev libsqlite3-dev \
+         libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+With dependencies installed you can install/compile runtimes:
+
+.. code:: bash
+
+  $ asdf install golang latest
+  $ asdf install nodejs latest
+  $ asdf install python latest
+  $ asdf install shellcheck latest
+
+Python will be compiled and will take a while.
+
+In the repository the version is defined in `.tool-versions`_. Outside the
+repository, its recommended that the runtime should use the versions of the OS
+(`Fallback to System Version`_) / if not already done register the system
+versions global:
+
+.. code:: bash
+
+   $ cd /
+   $ asdf global golang system
+   $ asdf global nodejs system
+   $ asdf global python system
+   $ asdf global shellcheck system
+
+.. _asdf: https://asdf-vm.com/
+.. _download asdf: https://asdf-vm.com/guide/getting-started.html#_2-download-asdf
+.. _install asdf: https://asdf-vm.com/guide/getting-started.html#_3-install-asdf
+.. _install plugins: https://asdf-vm.com/guide/getting-started.html#install-the-plugin
+.. _Fallback to System Version: https://asdf-vm.com/manage/versions.html#fallback-to-system-version

docs/admin/searx.botdetection.rst → docs/src/searx.botdetection.rst

@@ -12,8 +12,10 @@ Bot Detection
 .. automodule:: searx.botdetection
   :members:
 
-.. automodule:: searx.botdetection.limiter
-  :members:
+.. _botdetection ip_lists:
+
+IP lists
+========
 
 .. automodule:: searx.botdetection.ip_lists
   :members:
@@ -50,3 +52,11 @@ Probe HTTP headers
 
 .. automodule:: searx.botdetection.http_user_agent
   :members:
+
+.. _botdetection config:
+
+Config
+======
+
+.. automodule:: searx.botdetection.config
+  :members:

requirements-dev.txt

@@ -2,22 +2,22 @@ mock==5.1.0
 nose2[coverage_plugin]==0.14.0
 cov-core==1.15.0
 black==22.12.0
-pylint==3.0.1
+pylint==3.0.2
 splinter==0.19.0
-selenium==4.14.0
+selenium==4.15.2
 twine==4.0.2
 Pallets-Sphinx-Themes==2.1.1
 Sphinx<=7.1.2; python_version == '3.8'
 Sphinx==7.2.6; python_version > '3.8'
 sphinx-issues==3.0.1
 sphinx-jinja==2.0.2
-sphinx-tabs==3.4.1
+sphinx-tabs==3.4.4
 sphinxcontrib-programoutput==0.17
 sphinx-autobuild==2021.3.14
 sphinx-notfound-page==1.0.0
 myst-parser==2.0.0
-linuxdoc==20230827
+linuxdoc==20231020
 aiounittest==1.4.2
-yamllint==1.32.0
+yamllint==1.33.0
 wlc==1.13
 coloredlogs==15.0.1

requirements.txt

@@ -1,5 +1,5 @@
 certifi==2023.7.22
-babel==2.13.0
+babel==2.13.1
 flask-babel==4.0.0
 flask==3.0.0
 jinja2==3.1.2
@@ -9,11 +9,11 @@ python-dateutil==2.8.2
 pyyaml==6.0.1
 httpx[http2]==0.24.1
 Brotli==1.1.0
-uvloop==0.18.0
+uvloop==0.19.0
 httpx-socks[asyncio]==0.7.7
 setproctitle==1.3.3
 redis==4.6.0
 markdown-it-py==3.0.0
 typing_extensions==4.8.0
-fasttext-predict==0.9.2.1
+fasttext-predict==0.9.2.2
 pytomlpp==1.0.13

searx/__init__.py

@@ -108,5 +108,6 @@ def logging_config_debug():
 if settings['server']['public_instance']:
     logger.warning(
         "Be aware you have activated features intended only for public instances. "
-        + "This force the usage of the bot limiter and link_token plugins."
+        "This force the usage of the limiter and link_token / "
+        "see https://docs.searxng.org/admin/searx.limiter.html"
     )