Merge pull request ckan#47 from qld-gov-au/QOLDEV-323-ckan-2.10

[QOLDEV-323] remove CKAN 2.8 support

.circleci/build.sh

@@ -21,7 +21,7 @@ export DOCTOR_CHECK_SSH=0
 export DOCTOR_CHECK_WEBSERVER=0
 export DOCTOR_CHECK_BOOTSTRAP=0
 
-if [ "$CKAN_VERSION" = "2.8" ] || [ "$CKAN_VERSION" = "2.9-py2" ]; then
+if [ "$CKAN_VERSION" = "2.9-py2" ]; then
     PYTHON_VERSION=py2
 else
     PYTHON_VERSION=py3

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: [2.9, "2.9-py2", 2.8]
+        ckan-version: [2.9, "2.9-py2"]
 
     name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest

README.md

@@ -24,10 +24,10 @@ Data description and validation for CKAN with [Frictionless Data](https://fricti
      * [Changes in the metadata schema](#changes-in-the-metadata-schema)
      * [Extending via interfaces](#extending-via-interfaces)
   * [Action functions](#action-functions)
-	* [resource_validation_run](#resource_validation_run)
-	* [resource_validation_show](#resource_validation_show)
-	* [resource_validation_delete](#resource_validation_delete)
-	* [resource_validation_run_batch](#resource_validation_run_batch)
+    * [resource_validation_run](#resource_validation_run)
+    * [resource_validation_show](#resource_validation_show)
+    * [resource_validation_delete](#resource_validation_delete)
+    * [resource_validation_run_batch](#resource_validation_run_batch)
   * [Command Line Interface](#command-line-interface)
     * [Starting the validation process manually](#starting-the-validation-process-manually)
     * [Data validation reports](#data-validation-reports)
@@ -52,7 +52,7 @@ If you are eager to get started, jump to the [Installation](#installation) and [
 
 ## Versions supported and requirements
 
-This extension has been tested with CKAN 2.8 and 2.9.
+This extension has been tested with CKAN 2.9.
 
 It is strongly recommended to use it alongside [ckanext-scheming](https://github.com/ckan/ckanext-scheming) to define the necessary extra fields in the default CKAN schema. By default, the extension installs ckanext-scheming version 2.1.0.
 
@@ -76,14 +76,8 @@ Or:
 
 Create the database tables by running:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation init-db
 
-ON CKAN <= 2.8:
-
-    paster validation init-db -c ../path/to/ini/file
-
 
 ## Configuration
 
@@ -97,16 +91,16 @@ Once installed, add the `validation` plugin to the `ckan.plugins` configuration
 
 The extension requires changes in the CKAN metadata schema. The easiest way to add those is by using ckanext-scheming. Use these two configuration options to link to the dataset schema (replace with your own if you need to customize it) and the required presets:
 
-	scheming.dataset_schemas = ckanext.validation.examples:ckan_default_schema.json
-	scheming.presets = ckanext.scheming:presets.json
-    	               ckanext.validation:presets.json
+    scheming.dataset_schemas = ckanext.validation.examples:ckan_default_schema.json
+    scheming.presets = ckanext.scheming:presets.json
+                       ckanext.validation:presets.json
 
 Read more below about how to [change the CKAN metadata schema](#changes-in-the-metadata-schema)
 
 ### Operation modes
 Use the following to configure which queue async jobs are added to
 
-	ckanext.validation.queue = bulk (Defaults to default)
+    ckanext.validation.queue = bulk (Defaults to default)
 
 Use the following configuration options to choose the [operation modes](#operation-modes):
 
@@ -117,13 +111,13 @@ Use the following configuration options to choose the [operation modes](#operati
 
 By default validation will be run against the following formats: `CSV`, `XLSX` and `XLS`. You can modify these formats using the following option:
 
-	ckanext.validation.formats = csv xlsx
+    ckanext.validation.formats = csv xlsx
 
 You can also provide [validation options](#validation-options) that will be used by default when running the validation:
 
-	ckanext.validation.default_validation_options={
-	    "skip_checks": ["blank-rows", "duplicate-headers"],
-    	"headers": 3}
+    ckanext.validation.default_validation_options={
+        "skip_checks": ["blank-rows", "duplicate-headers"],
+        "headers": 3}
 
 Make sure to use indentation if the value spans multiple lines otherwise it won't be parsed.
 
@@ -302,14 +296,8 @@ This mode might be useful for instances where datasets are harvested from other
 
 You will need to run the `worker` commmand to pick up validation jobs. Please refer to the [background jobs documentation](http://docs.ckan.org/en/latest/maintaining/background-tasks.html) for more details:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file jobs worker
 
-ON CKAN <= 2.8:
-
-    paster jobs worker -c /path/to/ini/file
-
 
 #### Synchronous validation
 
@@ -577,51 +565,25 @@ def resource_validation_run_batch(context, data_dict):
 
 You can start (asynchronous) validation jobs from the command line using the `validation run` command. If no parameters are provided it will start a validation job for all resources in the site of suitable format (ie `ckanext.validation.formats`):
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation run
 
-ON CKAN <= 2.8:
-
-    paster validation run -c /path/to/ckan/ini
-
 You can limit the resources by specifying a dataset id or name:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation run -d statistical-data-2018
 
-ON CKAN <= 2.8:
-
-    paster validation run -c /path/to/ckan/ini -d statistical-data-2018
-
 Or providing arbitrary search parameters:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation run -s '{"fq":"res_format:XLSX"}'
 
-ON CKAN <= 2.8:
-
-    paster validation run -c ../ckan/development.ini -s '{"fq":"res_format:XLSX"}'
-
 
 ### Data validation reports
 
 The extension provides two small utilities to generate a global report with all the current data validation reports:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation report
 
     ckan -c /path/to/ini/file validation report-full
 
-ON CKAN <= 2.8:
-
-    paster validation report -c /path/to/ckan/ini
-
-    paster validation report-full -c /path/to/ckan/ini
-
 
 Both commands will print an overview of the total number of datasets and tabular resources, and a breakdown of how many have a validation status of success,
 failure or error. Additionally they will create a CSV report. `validation report` will create a report with all failing resources, including the following fields:
@@ -645,20 +607,14 @@ failure or error. Additionally they will create a CSV report. `validation report
 
 In both cases you can define the location of the output CSV passing the `-o` or `--output` option:
 
-ON CKAN >= 2.9:
-
     ckan -c /path/to/ini/file validation report-full -o /tmp/reports/validation_full.csv
 
-ON CKAN <= 2.8:
-
-	paster validation report-full -c /path/to/ckan/ini -o /tmp/reports/validation_full.csv
-
 
 Check the command help for more details:
 
-	validation --help
+    validation --help
 
-	Usage: validation [options] Utilities for the CKAN data validation extension
+    Usage: validation [options] Utilities for the CKAN data validation extension
 
     Usage:
         validation init-db
@@ -702,36 +658,36 @@ Check the command help for more details:
 
 
 
-	Options:
-	  -h, --help            show this help message and exit
-	  -v, --verbose
-	  -c CONFIG, --config=CONFIG
-							Config file to use.
-	  -f FILE_PATH, --file=FILE_PATH
-							File to dump results to (if needed)
-	  -y, --yes             Automatic yes to prompts. Assume "yes" as answer to
-							all prompts and run non-interactively
-	  -r RESOURCE_ID, --resource=RESOURCE_ID
-							 Run data validation on a particular resource (if the
-							format is suitable). It can be defined multiple times.
-							Not to be used with -d or -s
-	  -d DATASET_ID, --dataset=DATASET_ID
-							 Run data validation on all resources for a particular
-							dataset (if the format is suitable). You can use the
-							dataset id or name, and it can be defined multiple
-							times. Not to be used with -r or -s
-	  -s SEARCH_PARAMS, --search=SEARCH_PARAMS
-							Extra search parameters that will be used for getting
-							the datasets to run validation on. It must be a JSON
-							object like the one used by the `package_search` API
-							call. Supported fields are `q`, `fq` and `fq_list`.
-							Check the documentation for examples. Note that when
-							using this you will have to specify the resource
-							formats to target yourself. Not to be used with -r or
-							-d.
-	  -o OUTPUT_FILE, --output=OUTPUT_FILE
-							Location of the CSV validation report file on the
-							relevant commands.
+    Options:
+      -h, --help            show this help message and exit
+      -v, --verbose
+      -c CONFIG, --config=CONFIG
+                            Config file to use.
+      -f FILE_PATH, --file=FILE_PATH
+                            File to dump results to (if needed)
+      -y, --yes             Automatic yes to prompts. Assume "yes" as answer to
+                            all prompts and run non-interactively
+      -r RESOURCE_ID, --resource=RESOURCE_ID
+                             Run data validation on a particular resource (if the
+                            format is suitable). It can be defined multiple times.
+                            Not to be used with -d or -s
+      -d DATASET_ID, --dataset=DATASET_ID
+                             Run data validation on all resources for a particular
+                            dataset (if the format is suitable). You can use the
+                            dataset id or name, and it can be defined multiple
+                            times. Not to be used with -r or -s
+      -s SEARCH_PARAMS, --search=SEARCH_PARAMS
+                            Extra search parameters that will be used for getting
+                            the datasets to run validation on. It must be a JSON
+                            object like the one used by the `package_search` API
+                            call. Supported fields are `q`, `fq` and `fq_list`.
+                            Check the documentation for examples. Note that when
+                            using this you will have to specify the resource
+                            formats to target yourself. Not to be used with -r or
+                            -d.
+      -o OUTPUT_FILE, --output=OUTPUT_FILE
+                            Location of the CSV validation report file on the
+                            relevant commands.
 
 
 ## Running the Tests

ckanext/validation/common.py

@@ -7,8 +7,7 @@
 
 from ckantoolkit import (c, NotAuthorized,
                          ObjectNotFound, abort, _,
-                         render, get_action, config,
-                         check_ckan_version)
+                         render, get_action, config)
 
 from ckanext.validation import settings
 from ckanext.validation.logic.action import _search_datasets
@@ -63,12 +62,8 @@ def validation(resource_id, id=None):
 
 
 def user_confirm(msg):
-    if check_ckan_version(min_version='2.9'):
-        import click
-        return click.confirm(msg)
-    else:
-        from ckan.lib.cli import query_yes_no
-        return query_yes_no(msg) == 'yes'
+    import click
+    return click.confirm(msg)
 
 
 def error(msg):

ckanext/validation/helpers.py

@@ -3,8 +3,7 @@
 
 from six.moves.urllib.parse import urlparse
 from six import string_types
-from ckantoolkit import url_for, _, config, asbool,\
-    literal, check_ckan_version, h
+from ckantoolkit import url_for, _, config, asbool, literal, h
 
 
 def _get_helpers():
@@ -13,7 +12,6 @@ def _get_helpers():
         validation_extract_report_from_errors,
         dump_json_value,
         bootstrap_version,
-        is_ckan_29,
         validation_hide_source,
         is_url_valid
     )
@@ -49,10 +47,7 @@ def get_validation_badge(resource, in_listing=False):
     else:
         status = 'unknown'
 
-    if check_ckan_version(min_version='2.9.0'):
-        action = 'validation.read'
-    else:
-        action = 'validation_read'
+    action = 'validation.read'
 
     validation_url = url_for(
         action,
@@ -117,14 +112,6 @@ def bootstrap_version():
         return '2'
 
 
-def is_ckan_29():
-    """
-    Returns True if using CKAN 2.9+, with Flask and Webassets.
-    Returns False if those are not present.
-    """
-    return check_ckan_version(min_version='2.9.0')
-
-
 def validation_hide_source(type):
     """
     Returns True if the given source type must be hidden on form.

ckanext/validation/logic/action.py

@@ -98,14 +98,11 @@ def enqueue_validation_job(package_id, resource_id):
         }
     }
 
-    if tk.check_ckan_version('2.8'):
-        ttl = 24 * 60 * 60  # 24 hour ttl.
-        rq_kwargs = {
-            'ttl': ttl
-        }
-        if tk.check_ckan_version('2.9'):
-            rq_kwargs['failure_ttl'] = ttl
-        enqueue_args['rq_kwargs'] = rq_kwargs
+    ttl = 24 * 60 * 60  # 24 hour ttl.
+    rq_kwargs = {
+        'ttl': ttl, 'failure_ttl': ttl
+    }
+    enqueue_args['rq_kwargs'] = rq_kwargs
 
     # Optional variable, if not set, default queue is used
     queue = tk.config.get('ckanext.validation.queue', None)

ckanext/validation/plugin.py

@@ -9,21 +9,14 @@
 import ckan.plugins as p
 from ckan.lib.plugins import DefaultTranslation
 
-import ckanext.validation.settings as s
-from ckanext.validation.model import tables_exist
-from ckanext.validation.helpers import _get_helpers, is_ckan_29
-from ckanext.validation import validators
-from ckanext.validation import utils
-from ckanext.validation.logic import action
-from ckanext.validation.logic import auth
+from . import settings as s, utils, validators
+from .helpers import _get_helpers
+from .logic import action, auth
+from .model import tables_exist
+from .plugin_mixins.flask_plugin import MixinPlugin
 
 log = logging.getLogger(__name__)
 
-if is_ckan_29():
-    from .plugin_mixins.flask_plugin import MixinPlugin
-else:
-    from .plugin_mixins.pylons_plugin import MixinPlugin
-
 
 class ValidationPlugin(MixinPlugin, p.SingletonPlugin, DefaultTranslation):
     p.implements(p.IConfigurer)
@@ -47,10 +40,7 @@ def i18n_directory(self):
 
     def update_config(self, config_):
         if not tables_exist():
-            if is_ckan_29():
-                init_command = 'ckan validation init-db'
-            else:
-                init_command = 'paster --plugin=ckanext-validation validation init-db'
+            init_command = 'ckan validation init-db'
             log.critical(u'''
 The validation extension requires a database setup.
 Validation pages will not be enabled.

ckanext/validation/templates/package/resource_read.html

@@ -6,7 +6,7 @@ <h1 class="page-heading">{{ h.resource_display_name(res) | truncate(50) }}
   {{ h.get_validation_badge(res)|safe }}
   </h1>
 
-  {% set type = 'asset' if h.is_ckan_29() else 'resource' %}
+  {% set type = 'asset' %}
   {% include 'validation/snippets/validation_style_' ~ type ~ '.html' %}
 
 {% endblock %}

ckanext/validation/templates/package/snippets/resource_item.html

@@ -4,7 +4,7 @@
 {{ super() }}
 {{ h.get_validation_badge(res, in_listing=True)|safe }}
 
-{% set type = 'asset' if h.is_ckan_29() else 'resource' %}
+{% set type = 'asset' %}
 {% include 'validation/snippets/validation_style_' ~ type ~ '.html' %}
 {% endblock %}