Enhancements from Queensland Government fork

.flake8

@@ -0,0 +1,20 @@
+[flake8]
+# @see https://flake8.pycqa.org/en/latest/user/configuration.html?highlight=.flake8
+
+exclude =
+    ckan
+    scripts
+
+# Extended output format.
+format = pylint
+
+# Show the source of errors.
+show_source = True
+
+max-complexity = 10
+
+# List ignore rules one per line.
+ignore =
+    E501
+    C901
+    W503

.github/workflows/test.yml

@@ -0,0 +1,125 @@
+---
+#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml
+# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml
+name: Tests
+on: [push, pull_request]
+env:
+  CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+  CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+  CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+  CKAN_SOLR_URL: http://solr:8983/solr/ckan
+  CKAN_REDIS_URL: redis://redis:6379/1
+jobs:
+
+
+
+
+  lint:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+
+  test:
+    needs: lint
+    strategy:
+      matrix:
+        # not ready for CKAN 2.9 yet
+        # ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
+        ckan-version: [2.8, 2.7]
+        env:
+          - { ARCHIVER_GIT_REPO: "ckan", ARCHIVER_BRANCH: "master", REPORT_GIT_REPO: "datagovuk", REPORT_BRANCH: "master" }
+          - { ARCHIVER_GIT_REPO: "qld-gov-au", ARCHIVER_BRANCH: "2.1.0-qgov.1", REPORT_GIT_REPO: "qld-gov-au", REPORT_BRANCH: "0.1" }
+          - { ARCHIVER_GIT_REPO: "qld-gov-au", ARCHIVER_BRANCH: "develop", REPORT_GIT_REPO: "qld-gov-au", REPORT_BRANCH: "develop" }
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
+    runs-on: ubuntu-18.04
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+    services:
+      solr:
+        image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+          image: redis:3
+    env: ${{ matrix.env }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Install report and archiver plugins
+      run: |
+        echo "Installing dependency ckanext-report and its requirements..."
+        if [ ! -d ckanext-report ]; then
+          git clone --depth=50 --branch=$REPORT_BRANCH https://github.com/$REPORT_GIT_REPO/ckanext-report ckanext-report
+        fi
+        cd ckanext-report
+          if [ -f pip-requirements.txt ]; then
+            pip install -r pip-requirements.txt
+          fi
+          if [ -f dev-requirements.txt ]; then
+            pip install -r dev-requirements.txt
+          fi
+
+          if [ -f requirements.txt ]; then
+            pip install -r requirements.txt
+          fi
+          pip install --no-deps -e .
+        cd ..
+
+        echo "Installing dependency ckanext-archiver and its requirements..."
+        if [ ! -d ckanext-archiver ]; then
+          git clone --depth=50 --branch=$ARCHIVER_BRANCH https://github.com/$ARCHIVER_GIT_REPO/ckanext-archiver ckanext-archiver
+        fi
+        cd ckanext-archiver
+          if [ -f pip-requirements.txt ]; then
+            pip install -r pip-requirements.txt
+          fi
+          if [ -f dev-requirements.txt ]; then
+            pip install -r dev-requirements.txt
+          fi
+
+          if [ -f requirements.txt ]; then
+            pip install -r requirements.txt
+          fi
+          pip install --no-deps -e .
+        cd ..
+
+    - name: Install requirements
+      run: |
+        pip install -r dev-requirements.txt
+        if [ -f pip-requirements.txt ]; then
+          pip install -r pip-requirements.txt
+        fi
+        pip install -r requirements.txt
+        pip install -e .
+        apk add file
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+
+    - name: Setup extension (CKAN >= 2.9)
+      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+      run: |
+        ckan -c test.ini db init
+    - name: Setup extension (CKAN < 2.9)
+      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
+      run: |
+        paster --plugin=ckan db init -c test.ini
+    - name: Run all tests
+      run: |
+        nosetests --with-pylons=test.ini --with-coverage --cover-package=ckanext.qa --cover-inclusive --cover-erase --cover-tests
+
+

bin/travis-build.bash

@@ -6,34 +6,62 @@ echo "This is travis-build.bash..."
 
 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
-sudo apt-get install solr-jetty libcommons-fileupload-java
+sudo apt-get install -y solr-jetty libcommons-fileupload-java
+
+ver=$(python -c"import sys; print(sys.version_info.major)")
+if [ $ver -eq 2 ]; then
+    echo "python version 2"
+elif [ $ver -eq 3 ]; then
+    echo "python version 3"
+else
+    echo "Unknown python version: $ver"
+fi
 
 echo "Upgrading libmagic for ckanext-qa..."
 # appears to upgrade it from 5.09-2 to 5.09-2ubuntu0.6 which seems to help the tests
 sudo apt-get install libmagic1
 
 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
-cd ckan
+pushd ckan
+
+if [ -f requirement-setuptools.txt ]; then
+    pip install -r requirement-setuptools.txt
+fi
 
-if [ $CKANVERSION == 'master' ]
+if [ ${CKANVERSION:-master} == 'master' ]
 then
     echo "CKAN version: master"
+    export CKAN_MINOR_VERSION=100
 else
+    export CKAN_MINOR_VERSION=${CKANVERSION##*.}
     CKAN_TAG=$(git tag | grep ^ckan-$CKANVERSION | sort --version-sort | tail -n 1)
     git checkout $CKAN_TAG
     echo "CKAN version: ${CKAN_TAG#ckan-}"
 fi
 
-python setup.py develop
-if [ -f requirements-py2.txt ]
+if (( "$CKAN_MINOR_VERSION" >= 9 )) && (( $ver = 2 ))
 then
     pip install -r requirements-py2.txt
 else
     pip install -r requirements.txt
 fi
 pip install -r dev-requirements.txt --allow-all-external
-cd -
+python setup.py develop
+
+echo "Creating the PostgreSQL user and database..."
+sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+
+echo "Initialising the database..."
+if (( $CKAN_MINOR_VERSION >= 9 ))
+then
+    ckan -c test-core.ini db init
+else
+    paster db init -c test-core.ini
+fi
+
+popd
 
 echo "Setting up Solr..."
 # solr is multicore for tests on ckan master now, but it's easier to run tests
@@ -44,32 +72,35 @@ printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME"
 sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
 sudo service jetty restart
 
-echo "Creating the PostgreSQL user and database..."
-sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
-
-echo "Initialising the database..."
-cd ckan
-paster db init -c test-core.ini
-cd -
-
 echo "Installing dependency ckanext-report and its requirements..."
-pip install -e git+https://github.com/datagovuk/ckanext-report.git#egg=ckanext-report
+git clone --depth=50 https://github.com/datagovuk/ckanext-report.git
+pushd ckanext-report
+  if [ -f requirements-py2.txt ] && [ $ver -eq 2 ]; then
+    pip install -r requirements-py2.txt
+  elif [ -f requirements.txt ]; then
+    pip install -r requirements.txt
+  fi
+  pip install --no-deps -e .
+popd
 
 echo "Installing dependency ckanext-archiver and its requirements..."
-git clone https://github.com/ckan/ckanext-archiver.git
-cd ckanext-archiver
-pip install -e .
-pip install -r requirements.txt
-cd -
+git clone --depth=50 https://github.com/ckan/ckanext-archiver.git
+pushd ckanext-archiver
+  if [ -f requirements-py2.txt ] && [ $ver -eq 2 ]; then
+    pip install -r requirements-py2.txt
+  elif [ -f requirements.txt ]; then
+    pip install -r requirements.txt
+  fi
+  pip install --no-deps -e .
+popd
 
 echo "Installing ckanext-qa and its requirements..."
-python setup.py develop
 pip install -r requirements.txt
 pip install -r dev-requirements.txt
+python setup.py develop
 
 echo "Moving test-core.ini into a subdir..."
-mkdir subdir
-mv test-core.ini subdir
+mkdir -p subdir
+cp test-core.ini subdir
 
 echo "travis-build.bash is done."

bin/travis-run.sh

@@ -3,4 +3,4 @@
 echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
 sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
 sudo service jetty restart
-nosetests --with-pylons=subdir/test-core.ini --with-coverage --cover-package=ckanext.archiver --cover-inclusive --cover-erase --cover-tests
+nosetests --with-pylons=subdir/test-core.ini --with-coverage --cover-package=ckanext.qa --cover-inclusive --cover-erase --cover-tests

ckanext/qa/__init__.py

@@ -6,4 +6,4 @@
     import pkgutil
     __path__ = pkgutil.extend_path(__path__, __name__)
 
-__version__ = '2.0'
+__version__ = '2.1.0-rc1'

ckanext/qa/bin/common.py

@@ -48,5 +48,5 @@ def get_resources(state='active', publisher_ref=None, resource_id=None, dataset_
         resources = resources.filter(model.Resource.id == resource_id)
         criteria.append('Resource:%s' % resource_id)
     resources = resources.all()
-    print '%i resources (%s)' % (len(resources), ' '.join(criteria))
+    print('%i resources (%s)' % (len(resources), ' '.join(criteria)))
     return resources

ckanext/qa/bin/migrate_task_status.py

@@ -19,7 +19,7 @@
 
 START_OF_TIME = datetime.datetime(1980, 1, 1)
 END_OF_TIME = datetime.datetime(9999, 12, 31)
-TODAY = datetime.datetime.now()
+TODAY = datetime.datetime.utcnow()
 
 # NB put no CKAN imports here, or logging breaks
 
@@ -59,7 +59,7 @@ def migrate(options):
         # time, so some timezone nonesense going on. Can't do much.
         archival = Archival.get_for_resource(res.id)
         if not archival:
-            print add_stat('QA but no Archival data', res, stats)
+            print(add_stat('QA but no Archival data', res, stats))
             continue
         archival_date = archival.updated
         # the state of the resource was as it was archived on the date of
@@ -112,10 +112,10 @@ def migrate(options):
                 model.Session.add(qa)
             add_stat('Added to QA table', res, stats)
 
-    print 'Summary\n', stats.report()
+    print('Summary\n', stats.report())
     if options.write:
         model.repo.commit_and_remove()
-        print 'Written'
+        print('Written')
 
 
 def add_stat(outcome, res, stats, extra_info=None):
@@ -154,10 +154,10 @@ def date_str_to_datetime_or_none(date_str):
     if len(args) != 1:
         parser.error('Wrong number of arguments (%i)' % len(args))
     config_ini = args[0]
-    print 'Loading CKAN config...'
+    print('Loading CKAN config...')
     common.load_config(config_ini)
     common.register_translator()
-    print 'Done'
+    print('Done')
     # Setup logging to print debug out for local only
     rootLogger = logging.getLogger()
     rootLogger.setLevel(logging.WARNING)

ckanext/qa/bin/running_stats.py

@@ -14,7 +14,7 @@
         package_stats.increment('deleted')
     else:
         package_stats.increment('not deleted')
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30
 > not deleted: 70
 
@@ -26,7 +26,7 @@
         package_stats.add('deleted', package.name)
     else:
         package_stats.add('not deleted' package.name)
-print package_stats.report()
+print(package_stats.report())
 > deleted: 30 pollution-uk, flood-regions, river-quality, ...
 > not deleted: 70 spending-bristol, ...
 
@@ -42,7 +42,7 @@ class StatsCount(dict):
     report_value_limit = 150
 
     def __init__(self, *args, **kwargs):
-        self._start_time = datetime.datetime.now()
+        self._start_time = datetime.datetime.utcnow()
         super(StatsCount, self).__init__(*args, **kwargs)
 
     def _init_category(self, category):
@@ -80,7 +80,7 @@ def report(self, indent=1, order_by_title=False, show_time_taken=True):
             lines = [indent_str + 'None']
 
         if show_time_taken:
-            time_taken = datetime.datetime.now() - self._start_time
+            time_taken = datetime.datetime.utcnow() - self._start_time
             lines.append(indent_str + 'Time taken (h:m:s): %s' % time_taken)
         return '\n'.join(lines)
 
@@ -110,6 +110,6 @@ def report_value(self, category):
     package_stats.add('Success', 'good3')
     package_stats.add('Success', 'good4')
     package_stats.add('Failure', 'bad1')
-    print package_stats.report()
+    print(package_stats.report())
 
-    print StatsList().report()
+    print(StatsList().report())