[pull] master from ckan:master

.github/workflows/backports.yml

@@ -0,0 +1,73 @@
+name: Backport merged pull request
+on:
+  pull_request_target:
+    types: [closed]
+  issue_comment:
+    types: [created]
+permissions:
+  contents: write # so it can comment
+  pull-requests: write # so it can create pull requests
+jobs:
+  backport:
+    name: Backport pull request
+    runs-on: ubuntu-latest
+    # Only run when a pull request is merged
+    # or when a comment starting with `/backport` is created by a Tech Team member
+    if: |-
+      ${{
+        (
+          github.event_name == 'pull_request_target' &&
+          github.event.pull_request.merged
+        ) || (
+          github.event_name == 'issue_comment' &&
+          github.event.issue.pull_request &&
+          contains(fromJSON(vars.TECH_TEAM_USER_IDS), github.event.comment.user.id) &&
+          startsWith(github.event.comment.body, '/backport')
+        )
+      }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create backport pull requests
+        uses: korthout/backport-action@v2
+        with:
+          # Token to authenticate requests to GitHub. This is a Personal Access Token
+          # from the ckanbot user
+          github_token: ${{ secrets.BACKPORT_ACTION_PAT }}
+          # Run when there is one or more "Backport <branch>" labels,
+          # excluding "Backport pending"
+          label_pattern: "Backport (?!pending)([^ ]+)$"
+          merge_commits: skip
+          copy_assignees: true
+          pull_description: |-
+            This is an automated backport pull request 🏗️.
+
+            ### Details
+            |     |     |
+            | --- | --- |
+            | Original pull request | #${pull_number} (${pull_title}) |
+            | Original author | @${pull_author} |
+            | Target branch | **${target_branch}** |
+
+            Please make sure that all relevant checks pass before merging it.
+  backport-labels:
+    name: Backport failed labels
+    runs-on: ubuntu-latest
+    if: |-
+      ${{
+        github.event_name == 'issue_comment' &&
+        github.event.issue.pull_request &&
+        github.event.comment.user.id == vars.CKANBOT_USER_ID &&
+        startsWith(github.event.comment.body, 'Backport failed for')
+      }}
+    steps:
+      - name: Add Backport failed label to PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.BACKPORT_ACTION_PAT }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ["Backport failed"]
+            });

.readthedocs.yaml

@@ -9,9 +9,13 @@ version: 2
 build:
   os: ubuntu-22.04
   apt_packages:
-      - libmagic-dev
+    - libmagic-dev
+    - libmagic1
   tools:
     python: "3.10"
+  jobs:
+    post_checkout:
+      - git fetch --tags || true
 
 sphinx:
   configuration: doc/conf.py

CHANGELOG.rst

@@ -8,7 +8,169 @@ Changelog
 ---------
 
 .. towncrier release notes start
-
+
+v.2.10.4 2024-03-13
+===================
+
+Migration notes
+---------------
+
+- The default format for accepted uploads for user, groups and organization
+  images is now limited to PNG, GIF anf JPG. If you need to add additional
+  foramts you can use the :ref:`ckan.upload.user.mimetypes` and
+  :ref:`ckan.upload.group.mimetypes`) (`#7028
+  <https://github.com/ckan/ckan/pull/7028>`_)
+- Public user registration is disabled by default, ie users can not create
+  new accounts from the UI. With this default value, new users can be created
+  by being invited by an organization admin, being created directly by a
+  sysadmin in the ``/user/register`` endpoint  or being created in the CLI
+  using ``ckan user add``. To allow public registration see
+  :ref:`ckan.auth.create_user_via_web`, but it's strongly encouraged to put
+  some measures in place to avoid spam. (`#7028
+  <https://github.com/ckan/ckan/pull/7028>`_) (`#7208
+  <https://github.com/ckan/ckan/pull/7208>`_)
+
+Minor changes
+-------------
+- Define allowed alternative Solr query parsers via the :ref:`ckan.search.solr_allowed_query_parsers`
+  config option (`#8053 <https://github.com/ckan/ckan/pull/8053>`_)
+
+Bugfixes
+--------
+- `CVE-2024-27097 <https://github.com/ckan/ckan/security/advisories/GHSA-8g38-3m6v-232j>`_: fixed
+  potential log injection in reset user endpoint.
+- use custom group type from the activity object if it's not supplied, eg on
+  user activity streams (`#7980 <https://github.com/ckan/ckan/pull/7980>`_)
+- Removes extra <<<HEAD from resources list template (`#7998
+  <https://github.com/ckan/ckan/pull/7998>`_)
+- CKAN does not start without ``beaker.session.validate_key`` option introduced
+  in v2.10.3 (`#8023 <https://github.com/ckan/ckan/pull/8023>`_)
+- Editing of resources unavailable from package view page. (`#8025
+  <https://github.com/ckan/ckan/pull/8025>`_)
+- Pass custom package types through to the 'new resource' activity item (`#8034
+  <https://github.com/ckan/ckan/pull/8034>`_)
+- Fix Last Modified sort parameter for bulk-process page (`#8048
+  <https://github.com/ckan/ckan/pull/8048>`_)
+- Detect XLSX mimetypes correctly in uploader (`#8088
+  <https://github.com/ckan/ckan/pull/8088>`_)
+- Remove nginx cache as configuration from documentation (`#8031
+  <https://github.com/ckan/ckan/pull/8031>`_)
+- Fix `clean_db` fixtures breaking when tables are missing (`#8054
+  <https://github.com/ckan/ckan/pull/8054>`_)
+- Fix JS error in flash message when adding a Member (`#8104
+  <https://github.com/ckan/ckan/pull/8104>`_)
+
+
+v.2.10.3 2023-12-13
+===================
+
+
+Minor changes
+-------------
+- New sites now default to cookie-based sessions (the default value for ``beaker.session.type``
+  is now ``cookie``. The ``beaker.session.samesite`` configuration option has been introduced,
+  allowing you to specify the ``SameSite`` attribute for session cookies. This attribute determines
+  how cookies are sent in cross-origin requests, enhancing security and privacy.
+
+  .. note:: When using cookie-based sessions, it is now required to
+    set ``beaker.session.validate_key`` appropriately.
+
+- Skip interactive mode of ``ckan user setpass`` using ``-p``/``--password``
+  option. (`#7530 <https://github.com/ckan/ckan/pull/7530>`_)
+- Added support for Solr 9. Users of the `official Docker images
+  <https://github.com/ckan/ckan-solr>`_ can use the
+  ``ckan/ckan-solr:2.10-solr9`` tag. (`#7693
+  <https://github.com/ckan/ckan/pull/7693>`_)
+- Update requirements to support more Python versions (`#7935
+  <https://github.com/ckan/ckan/issues/7935>`_)
+- Add tooltips when links are truncated, to show the full text. (`#7743
+  <https://github.com/ckan/ckan/pull/7743>`_)
+- Added pages to confirm User delete and Dataset Collaborator delete.
+  Fixed cancellation of Group Member delete. (`#7813
+  <https://github.com/ckan/ckan/pull/7813>`_)
+- The ``validators`` attribute of a declared config option makes tries to parse
+  arguments to validators as python literals. If **all** arguments can be
+  parsed, they are passed to a validator factory with original types. If at least one
+  argument is not a valid Python literal, all values are passed as a string
+  (this was the previous behavior). Space characters are still not allowed inside
+  arguments, use the ``\\x20`` symbol if you need a space in a literal (`#7615
+  <https://github.com/ckan/ckan/pull/7615>`_)::
+
+      # Not changed
+      `validators: v(xxx)` # v("xxx")
+      `validators: v("xxx",yyy)` # v("xxx", "yyy")
+      `validators: v(1,2,none)` # v("1", "2", "none")
+      `validators: v("hello\\x20world")` # v("hello world")
+
+      # Changed
+      `validators: v("xxx")` # v("xxx")
+      `validators: v("xxx",1)` # v("xxx", 1)
+      `validators: v(1,2,None)` # v(1, 2, None)
+
+- Automatically add the ``not_empty`` validator to any config option declared
+  with ``required: true`` (`#7658 <https://github.com/ckan/ckan/pull/7658>`_)
+
+
+Bugfixes
+--------
+- `CVE-2023-50248 <https://github.com/ckan/ckan/security/advisories/GHSA-7fgc-89cx-w8j5>`_: fix potential
+  out of memory error when submitting the dataset form with a specially-crafted field.
+- Fix ``deprecated`` decorator (`#7939
+  <https://github.com/ckan/ckan/pull/7939>`_)
+- Fix for missing Tag facets on Home page (`#7520
+  <https://github.com/ckan/ckan/pull/7520>`_)
+- Fix errors when running the `ckan db upgrade` command (`#7681
+  <https://github.com/ckan/ckan/pull/7681>`_)
+- Fix datastore_search + downloading datastore resources as json with null
+  values (`#6713 <https://github.com/ckan/ckan/pull/6713>`_)
+- ``CONFIG_FROM_ENV_VARS`` takes precedence over config file and extensions but
+  those settings are not normalized. (`#7502
+  <https://github.com/ckan/ckan/pull/7502>`_)
+- Fixed server not recognizing SSL settings in configuration .ini file
+  (`#7758 <https://github.com/ckan/ckan/pull/7758>`_)
+- Fix error when indexing a full ISO date with timezone info (`#7775
+  <https://github.com/ckan/ckan/pull/7775>`_)
+- Aligned `member_create` with `group_member_save` to prevent possible member
+  duplication. (`#7804 <https://github.com/ckan/ckan/pull/7804>`_)
+- datastore-only resources now have a visible download button on the resource
+  page (`#7806 <https://github.com/ckan/ckan/pull/7806>`_)
+- update resource ``datastore_active`` with a single statement on
+  ``datastore_create/delete`` (`#7832 <https://github.com/ckan/ckan/pull/7832>`_)
+- Fixed Octet Streaming for Datastore Dump requests. (`#7839
+  <https://github.com/ckan/ckan/pull/7839>`_)
+- Fixed restricting anonymous users in actions to check user in context.
+  (`#7871 <https://github.com/ckan/ckan/pull/7871>`_)
+- Empty string in ``beaker.session.timeout`` produces an error instead of
+  never-expiring session (`#7881 <https://github.com/ckan/ckan/pull/7881>`_)
+- Updated Bootstrap alert-error class to alert-danger (`#7901
+  <https://github.com/ckan/ckan/pull/7901>`_)
+- Changed dataset query to check for ``+state:`` in the ``fq_list`` as well as the
+  `fq` parameter before forcing ``state:active`` (`#7905
+  <https://github.com/ckan/ckan/pull/7905>`_)
+- View modules use pluggable ``ckan.plugins.toolkit.h`` instead of
+  `ckan.lib.helpers` (`#7923 <https://github.com/ckan/ckan/pull/7923>`_)
+- Fix HTML5 validation failing on resource uploads (`#7925
+  <https://github.com/ckan/ckan/pull/7925>`_)
+- Fixed issues with the ``ckan views create`` CLI sub-command. (`#7944
+  <https://github.com/ckan/ckan/pull/7944>`_)
+- Improve handling of date fields in Solr (`#7775
+  <https://github.com/ckan/ckan/pull/7775>`_)
+- Fix URL validator does not support ":" for specifying ports (`#7891
+  <https://github.com/ckan/ckan/pull/7891>`_)
+- Fix user_show for ``ckan.auth.public_user_details`` (`#7866
+  <https://github.com/ckan/ckan/pull/7866>`_)
+- Add missing translations to aria-label attributes (`#7947
+  <https://github.com/ckan/ckan/pull/7947>`_)
+- Catch AttributeErrors in license retrieval (`#7931
+  <https://github.com/ckan/ckan/pull/7948>`_)
+- Fix downloading datastore resources as json with null values in json columns
+  (`#7545 <https://github.com/ckan/ckan/pull/7545>`_)
+
+v.2.10.2
+========
+
+Unreleased
+
 v.2.10.1 2023-05-24
 ===================
 
@@ -570,6 +732,45 @@ Removals and deprecations
 - ``ckan.route_after_login`` renamed to ``ckan.auth.route_after_login`` (`#7350
   <https://github.com/ckan/ckan/pull/7350>`_)
 
+v.2.9.11 2024-03-13
+===================
+
+Minor changes
+-------------
+- Define allowed alternative Solr query parsers via the :ref:`ckan.search.solr_allowed_query_parsers`
+  config option (`#8053 <https://github.com/ckan/ckan/pull/8053>`_). Note that the 2.9 version of this
+  patch does not use pyparsing to parse the local parameters string, so some limitations are in place,
+  mainly that no quotes are allowed in the local paramaters definition.
+- Get default formats for DataStore views from config (`#8095 <https://github.com/ckan/ckan/pull/8095>`_)
+
+Bugfixes
+--------
+- `CVE-2024-27097 <https://github.com/ckan/ckan/security/advisories/GHSA-8g38-3m6v-232j>`_: fixed
+  potential log injection in reset user endpoint.
+- Fixed Octet Streaming for Datastore Dump requests. (`#7899 <https://github.com/ckan/ckan/pull/7899>`_)
+- Fix Password Reset Keys with multiple accounts (`#8079 <https://github.com/ckan/ckan/pull/8079>`_)
+- Detect XLSX mimetypes correctly in uploader (`#8088 <https://github.com/ckan/ckan/pull/8088>`_)
+
+
+v.2.9.10 2023-12-13
+===================
+
+Bugfixes
+--------
+
+- `CVE-2023-50248 <https://github.com/ckan/ckan/security/advisories/GHSA-7fgc-89cx-w8j5>`_: fix potential
+  out of memory error when submitting the dataset form with a specially-crafted field.
+- Update resource datastore_active with a single statement (`#7833 <https://github.com/ckan/ckan/pull/7833>`_)
+- Fix downloading datastore resources as json with null values in json columns
+  (`#7545 <https://github.com/ckan/ckan/pull/7545>`_)
+- Fix errors when running the `ckan db upgrade` command (`#7681
+  <https://github.com/ckan/ckan/pull/7681>`_)
+- Fix ``deprecated`` decorator (`#7939
+  <https://github.com/ckan/ckan/pull/7939>`_)
+- Changed dataset query to check for ``+state:`` in the ``fq_list`` as well as the
+  `fq` parameter before forcing ``state:active`` (`#7905
+  <https://github.com/ckan/ckan/pull/7905>`_)
+
 v.2.9.9 2023-05-24
 ==================
 

changes/4193.feature

@@ -0,0 +1 @@
+Allow sysadmins to change usernames of other accounts

changes/6118.feature

@@ -0,0 +1,10 @@
+Table Designer: UI for datastore-first datasets
+
+Use Table Designer option on resource url/upload control for:
+- automatic creation of datatable view for new Table Designer resources
+- add/delete columns and edit schema via Data Dictionary page
+- primary keys and required columns fully supported
+- add individual rows with an auto-generated form based on the schema
+- data validation enforced by postgresql triggers, rendered as friendly errors in forms
+- extended datatable preview with "edit row" and "delete rows" buttons for managing data
+- automatic API documentation for upsert/delete with examples from real data when available

changes/6146.feature

@@ -0,0 +1 @@
+Render snippets faster through better use of existing jinja2 tags. Use ``{% snippet 'path/to/snippet.html', arg1=test %}`` instead of ``{{ h.snippet('path/to/snippet.html', arg1=test) }}`` in templates for better performance.

changes/6254.feature

@@ -0,0 +1 @@
+`ckan.devserver.watch_patterns` config option supports glob patterns

changes/7402.feature

@@ -0,0 +1 @@
+update documenatation for CKAN SHELL command.

changes/7631.feature

@@ -0,0 +1 @@
+`user_with_token` / `sysadmin_with_token` factory fixtures added to test suite.

changes/7893.removal

@@ -0,0 +1,18 @@
+The sessions handling has been refactored, dropping the Beaker library in favour of
+`Flask-Session <https://flask-session.readthedocs.io/en/latest/config.html>`_.
+Note that the default session backend for new sites remains the client-side browser
+cookie based. See :ref:`SESSION_TYPE` for alternative backends available.
+The following configuration options need to be updated:
+
+================================= ==============================================
+Old configuration key             New configuration key
+================================= ==============================================
+``beaker.session.type``           :ref:`SESSION_TYPE`
+``beaker.session.key``            :ref:`SESSION_COOKIE_NAME`
+``beaker.session.cookie_expires`` :ref:`SESSION_PERMANENT` (with opposite value)
+``beaker.session.timeout``        :ref:`PERMANENT_SESSION_LIFETIME`
+``beaker.session.cookie_domain``  :ref:`SESSION_COOKIE_DOMAIN`
+``beaker.session.secure``         :ref:`SESSION_COOKIE_SECURE`
+``beaker.session.httponly``       :ref:`SESSION_COOKIE_HTTPONLY`
+``beaker.session.samesite``       :ref:`SESSION_COOKIE_SAMESITE`
+================================= ==============================================

changes/7927.removal

@@ -0,0 +1,2 @@
+Move datastore-specific download logic from ckan/templates/package/resource_read.html
+to ckanext/datastore/templates/package/resource_read.html

changes/7932.feature

@@ -0,0 +1,2 @@
+Introducing a new parameter to the user_create action - with_apitoken. 
+When set, this parameter triggers the creation of an API token for the user.

changes/7971.feature

@@ -0,0 +1,8 @@
+IDataDictionaryForm for extending and validating new keys in the `fields`
+dicts. Unlike the `info` free-form dict these new keys are possible to
+tightly control with a schema. The schema is built by combining schemas
+from from all plugins implementing this interface so plugins implementing
+different features may all contribute to the same schema.
+
+The underlying storage for data dictionary fields has changed. Use:
+`ckan datastore upgrade` after upgrading to this release.

changes/7976.removal

@@ -0,0 +1,35 @@
+- PyUtilib dependency removed. All the primitives for the plugin system are now defined in CKAN.
+- The deprecated methods with the form ``after_<action>`` and ``before_<action>`` of the :py:class:`~ckan.plugins.interfaces.IPackageController` and :py:class:`~ckan.plugins.interfaces.IResourceController` interfaces have been removed. The form ``after_<type>_<action>`` must be used from now on. E.g. ``after_create()`` -> ``after_dataset_create()`` or ``after_resource_create()``.
+- It is now possible to extend interface classes directly when implementing plugins, which provides better integration with development tools, e.g.::
+
+    class Plugin(p.SingletonPlugin, IClick):
+        pass
+
+ This is equivalent to::
+
+    class Plugin(p.SingletonPlugin):
+        p.implements(p.IClick, inherit=True)
+
+- All plugins need to be instances of p.SingletonPlugin, they can't inherit
+  from a base class that is an instance itself. For example, you need to move
+  from this::
+
+    class FirstPlugin(p.SingletonPlugin):
+        p.implements(ISomething)
+        def some_method(self):
+            pass
+
+    class SecondPlugin(FirstPlugin):
+        p.implements(IAnything)
+
+To this::
+
+    class BasePlugin():
+        def some_method(self):
+            pass
+
+    class FirstPlugin(p.SingletonPlugin, BasePlugin):
+        p.implements(ISomething)
+
+    class SecondPlugin(p.SingletonPlugin, BasePlutin):
+        p.implements(IAnything)

changes/7980.bugfix

@@ -0,0 +1 @@
+use custom group type from the activity object if it's not supplied, eg on user activity streams