From 299798f758df76effc587dd7b7b3bfe58460e9ea Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:31:47 -0400 Subject: [PATCH 01/21] Update pip dependencies --- dependencies/pip/dev_requirements.txt | 241 ++++++++++++-------------- dependencies/pip/requirements.in | 10 +- dependencies/pip/requirements.txt | 183 ++++++++++--------- 3 files changed, 207 insertions(+), 227 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 6845fa1e9..ad11ba548 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -6,52 +6,51 @@ # -e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/kobo-service-account.git@871762cdd099ed543d36f0a29bfbff1de4766a8b#egg=kobo-service-account +-e git+https://github.com/kobotoolbox/kobo-service-account.git@8ee76730106ff8dc0ee2539c8e6a567aea4ed9ee#egg=kobo-service-account # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==5.1.1 +amqp==5.2.0 # via # -r dependencies/pip/requirements.in # kombu -asgiref==3.5.0 - # via django -asttokens==2.0.5 +asgiref==3.7.2 + # via + # django + # django-cors-headers +asttokens==2.4.1 # via stack-data -async-timeout==4.0.2 +async-timeout==4.0.3 # via redis -attrs==21.4.0 - # via pytest -azure-core==1.23.1 - # via azure-storage-blob -azure-storage-blob==12.11.0 +azure-core==1.30.1 + # via + # azure-storage-blob + # django-storages +azure-storage-blob==12.19.1 # via django-storages -backcall==0.2.0 - # via ipython -billiard==3.6.4.0 +billiard==4.2.0 # via celery -boto3==1.22.2 +boto3==1.34.60 # via django-storages -botocore==1.25.2 +botocore==1.34.60 # via # boto3 # s3transfer -celery[redis]==5.2.6 +celery[redis]==5.3.6 # via # -r dependencies/pip/requirements.in # django-celery-beat -certifi==2021.10.8 +certifi==2024.2.2 # via - # msrest # requests # sentry-sdk -cffi==1.15.0 +cffi==1.16.0 # via cryptography -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via requests -click==8.1.2 +click==8.1.7 # via # celery # click-didyoumean @@ -61,9 +60,11 @@ click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery -cryptography==37.0.0 +cron-descriptor==1.4.3 + # via django-celery-beat +cryptography==42.0.5 # via # azure-storage-blob # jwcrypto @@ -76,13 +77,9 @@ defusedxml==0.7.1 # -r dependencies/pip/requirements.in # djangorestframework-xml # pyxform -deprecated==1.2.13 - # via - # jwcrypto - # redis -dict2xml==1.7.1 +dict2xml==1.7.5 # via -r dependencies/pip/requirements.in -django==3.2.15 +django==4.2.11 # via # -r dependencies/pip/requirements.in # django-celery-beat @@ -103,53 +100,53 @@ django==3.2.15 # djangorestframework-guardian # jsonfield # kobo-service-account -django-celery-beat==2.2.1 +django-celery-beat==2.6.0 # via -r dependencies/pip/requirements.in -django-cors-headers==3.11.0 +django-cors-headers==4.3.1 # via -r dependencies/pip/requirements.in -django-csp==3.7 +django-csp==3.8 # via -r dependencies/pip/requirements.in django-db-readonly==0.7.0 # via -r dependencies/pip/requirements.in -django-environ==0.8.1 +django-environ==0.11.2 # via -r dependencies/pip/requirements.in -django-extensions==3.1.5 +django-extensions==3.2.3 # via -r dependencies/pip/requirements.in -django-filter==21.1 +django-filter==24.1 # via -r dependencies/pip/requirements.in django-guardian==2.4.0 # via # -r dependencies/pip/requirements.in # djangorestframework-guardian -django-oauth-toolkit==2.0.0 +django-oauth-toolkit==2.3.0 # via -r dependencies/pip/requirements.in django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis==5.2.0 +django-redis==5.4.0 # via -r dependencies/pip/requirements.in django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in -django-render-block==0.9.1 +django-render-block==0.9.2 # via django-templated-email -django-reversion==5.0.0 +django-reversion==5.0.12 # via -r dependencies/pip/requirements.in -django-storages[azure,boto3]==1.13.2 +django-storages[azure,boto3]==1.14.2 # via -r dependencies/pip/requirements.in -django-taggit==2.1.0 +django-taggit==5.0.1 # via -r dependencies/pip/requirements.in -django-templated-email==3.0.0 +django-templated-email==3.0.1 # via -r dependencies/pip/requirements.in -django-timezone-field==4.2.3 +django-timezone-field==6.1.0 # via # -r dependencies/pip/requirements.in # django-celery-beat -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via # -r dependencies/pip/requirements.in # djangorestframework-csv # djangorestframework-guardian # kobo-service-account -djangorestframework-csv==2.1.1 +djangorestframework-csv==3.0.2 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in @@ -157,213 +154,203 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -dpath==2.0.6 +dpath==2.1.6 # via -r dependencies/pip/requirements.in -ecdsa==0.17.0 +ecdsa==0.18.0 # via tlslite-ng elaphe3==0.2.0 # via -r dependencies/pip/requirements.in et-xmlfile==1.1.0 # via openpyxl -executing==0.8.3 +exceptiongroup==1.2.0 + # via + # ipython + # pytest +executing==2.0.1 # via stack-data gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in httmock==1.4.0 # via -r dependencies/pip/dev_requirements.in -idna==3.3 +idna==3.6 # via requests -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest -ipdb==0.13.9 +ipdb==0.13.13 # via -r dependencies/pip/dev_requirements.in -ipython==8.2.0 +ipython==8.22.2 # via # -r dependencies/pip/dev_requirements.in # ipdb isodate==0.6.1 - # via msrest -jedi==0.18.1 + # via azure-storage-blob +jedi==0.19.1 # via ipython -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -jwcrypto==1.0 +jwcrypto==1.5.6 # via django-oauth-toolkit -kombu==5.2.4 +kombu==5.3.5 # via celery -lxml==4.8.0 +lxml==5.1.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.3.6 +markdown==3.5.2 # via -r dependencies/pip/requirements.in -matplotlib-inline==0.1.3 +markupsafe==2.1.5 + # via werkzeug +matplotlib-inline==0.1.6 # via ipython -mock==4.0.3 +mock==5.1.0 # via -r dependencies/pip/dev_requirements.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -mongomock==4.0.0 +mongomock==4.1.2 # via -r dependencies/pip/dev_requirements.in -msrest==0.6.21 - # via azure-storage-blob -numpy==1.22.3 +numpy==1.26.4 # via pandas -oauthlib==3.2.0 - # via - # django-oauth-toolkit - # requests-oauthlib +oauthlib==3.2.2 + # via django-oauth-toolkit openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -packaging==21.3 +packaging==24.0 # via # mongomock # pytest - # redis -pandas==1.4.2 +pandas==2.2.1 # via -r dependencies/pip/requirements.in parso==0.8.3 # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 +pexpect==4.9.0 # via ipython -pillow==9.1.0 +pillow==10.2.0 # via # -r dependencies/pip/requirements.in # elaphe3 -pluggy==1.0.0 +pluggy==1.4.0 # via pytest -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.43 # via # click-repl # ipython -psycopg2==2.9.3 +psycopg==3.1.18 # via -r dependencies/pip/requirements.in ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -py==1.11.0 - # via pytest pycparser==2.21 # via cffi -pygments==2.12.0 +pygments==2.17.2 # via ipython -pymongo==3.12.3 +pymongo==3.13.0 # via -r dependencies/pip/requirements.in -pyparsing==3.0.8 - # via packaging -pytest==7.1.2 +pytest==8.1.1 # via # -r dependencies/pip/dev_requirements.in # pytest-django # pytest-env -pytest-django==4.5.2 +pytest-django==4.8.0 # via -r dependencies/pip/dev_requirements.in -pytest-env==0.6.2 +pytest-env==1.1.3 # via -r dependencies/pip/dev_requirements.in -python-crontab==2.6.0 +python-crontab==3.0.0 # via django-celery-beat -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore + # celery # pandas # python-crontab -pytz==2022.1 +pytz==2024.1 # via - # celery - # django - # django-timezone-field # djangorestframework # pandas pyxform==1.9.0 # via -r dependencies/pip/requirements.in -redis==4.2.2 +redis==5.0.3 # via # -r dependencies/pip/requirements.in # celery # django-redis # django-redis-sessions # kobo-service-account -requests==2.27.1 +requests==2.31.0 # via # azure-core # django-oauth-toolkit # httmock - # msrest - # requests-oauthlib -requests-oauthlib==1.3.1 - # via msrest -s3transfer==0.5.2 +s3transfer==0.10.0 # via boto3 sentinels==1.0.0 # via mongomock -sentry-sdk==1.5.10 +sentry-sdk==1.41.0 # via -r dependencies/pip/requirements.in shell-command==0.1 # via -r dependencies/pip/dev_requirements.in -simplejson==3.17.6 +simplejson==3.19.2 # via -r dependencies/pip/requirements.in six==1.16.0 # via # asttokens # azure-core - # click-repl - # djangorestframework-csv # ecdsa # isodate - # mongomock # python-dateutil -sqlparse==0.4.2 +sqlparse==0.4.4 # via # -r dependencies/pip/dev_requirements.in # django -stack-data==0.2.0 +stack-data==0.6.3 # via ipython tlslite-ng==0.7.6 # via gdata-python3 -toml==0.10.2 - # via ipdb tomli==2.0.1 - # via pytest -traitlets==5.1.1 + # via + # ipdb + # pytest + # pytest-env +traitlets==5.14.2 # via # ipython # matplotlib-inline -typing-extensions==4.2.0 - # via azure-core -unicodecsv==0.14.1 - # via djangorestframework-csv -urllib3==1.26.9 +typing-extensions==4.10.0 + # via + # asgiref + # azure-core + # azure-storage-blob + # jwcrypto + # psycopg +tzdata==2024.1 + # via + # celery + # django-celery-beat + # pandas +urllib3==2.0.7 # via # botocore # requests # sentry-sdk -uwsgi==2.0.21 +uwsgi==2.0.24 # via -r dependencies/pip/requirements.in -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.5 +wcwidth==0.2.13 # via prompt-toolkit -werkzeug==2.0.3 +werkzeug==3.0.1 # via -r dependencies/pip/requirements.in -wrapt==1.14.0 - # via deprecated xlrd==2.0.1 # via pyxform xlwt==1.3.0 # via -r dependencies/pip/requirements.in - -# The following packages are considered to be unsafe in a requirements file: -# setuptools backports-zoneinfo==0.2.1; python_version < '3.9' diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index c1d73c31f..c72fce580 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -2,7 +2,7 @@ # https://github.com/bndr/pipreqs is a handy utility, too. # service-account --e git+https://github.com/kobotoolbox/kobo-service-account.git@871762cdd099ed543d36f0a29bfbff1de4766a8b#egg=kobo-service-account +-e git+https://github.com/kobotoolbox/kobo-service-account.git@8ee76730106ff8dc0ee2539c8e6a567aea4ed9ee#egg=kobo-service-account # More up-to-date version of django-digest than PyPI seems to have. # Also, python-digest is an unlisted dependency thereof. @@ -14,7 +14,7 @@ # Regular PyPI packages defusedxml -Django>=3.2,<3.3 +Django>=4.2,<4.3 django-csp django-environ django-redis @@ -24,8 +24,8 @@ dpath gdata-python3 modilabs-python-utils Pillow -psycopg2 -pymongo==3.12.3 +psycopg +pymongo>=3.12,< 4.0 lxml pyxform==1.9.0 django-reversion @@ -82,7 +82,7 @@ djangorestframework-guardian sentry-sdk # mimetype detection -Werkzeug<=2.0.3 +Werkzeug # <=2.0.3 # WSGI server uWSGI diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 108ada2df..b63d48d42 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -6,46 +6,49 @@ # -e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/kobo-service-account.git@871762cdd099ed543d36f0a29bfbff1de4766a8b#egg=kobo-service-account +-e git+https://github.com/kobotoolbox/kobo-service-account.git@8ee76730106ff8dc0ee2539c8e6a567aea4ed9ee#egg=kobo-service-account # via -r dependencies/pip/requirements.in -e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest # via -r dependencies/pip/requirements.in -e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect # via -r dependencies/pip/requirements.in -amqp==5.1.1 +amqp==5.2.0 # via # -r dependencies/pip/requirements.in # kombu -asgiref==3.5.0 - # via django -async-timeout==4.0.2 +asgiref==3.7.2 + # via + # django + # django-cors-headers +async-timeout==4.0.3 # via redis -azure-core==1.23.1 - # via azure-storage-blob -azure-storage-blob==12.11.0 +azure-core==1.30.1 + # via + # azure-storage-blob + # django-storages +azure-storage-blob==12.19.1 # via django-storages -billiard==3.6.4.0 +billiard==4.2.0 # via celery -boto3==1.22.2 +boto3==1.34.60 # via django-storages -botocore==1.25.2 +botocore==1.34.60 # via # boto3 # s3transfer -celery[redis]==5.2.6 +celery[redis]==5.3.6 # via # -r dependencies/pip/requirements.in # django-celery-beat -certifi==2021.10.8 +certifi==2024.2.2 # via - # msrest # requests # sentry-sdk -cffi==1.15.0 +cffi==1.16.0 # via cryptography -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via requests -click==8.1.2 +click==8.1.7 # via # celery # click-didyoumean @@ -55,9 +58,11 @@ click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 # via celery -click-repl==0.2.0 +click-repl==0.3.0 # via celery -cryptography==37.0.0 +cron-descriptor==1.4.3 + # via django-celery-beat +cryptography==42.0.5 # via # azure-storage-blob # jwcrypto @@ -66,13 +71,9 @@ defusedxml==0.7.1 # -r dependencies/pip/requirements.in # djangorestframework-xml # pyxform -deprecated==1.2.13 - # via - # jwcrypto - # redis -dict2xml==1.7.1 +dict2xml==1.7.5 # via -r dependencies/pip/requirements.in -django==3.2.15 +django==4.2.11 # via # -r dependencies/pip/requirements.in # django-celery-beat @@ -93,53 +94,53 @@ django==3.2.15 # djangorestframework-guardian # jsonfield # kobo-service-account -django-celery-beat==2.2.1 +django-celery-beat==2.6.0 # via -r dependencies/pip/requirements.in -django-cors-headers==3.11.0 +django-cors-headers==4.3.1 # via -r dependencies/pip/requirements.in -django-csp==3.7 +django-csp==3.8 # via -r dependencies/pip/requirements.in django-db-readonly==0.7.0 # via -r dependencies/pip/requirements.in -django-environ==0.8.1 +django-environ==0.11.2 # via -r dependencies/pip/requirements.in -django-extensions==3.1.5 +django-extensions==3.2.3 # via -r dependencies/pip/requirements.in -django-filter==21.1 +django-filter==24.1 # via -r dependencies/pip/requirements.in django-guardian==2.4.0 # via # -r dependencies/pip/requirements.in # djangorestframework-guardian -django-oauth-toolkit==2.0.0 +django-oauth-toolkit==2.3.0 # via -r dependencies/pip/requirements.in django-pure-pagination==0.3.0 # via -r dependencies/pip/requirements.in -django-redis==5.2.0 +django-redis==5.4.0 # via -r dependencies/pip/requirements.in django-redis-sessions==0.6.2 # via -r dependencies/pip/requirements.in -django-render-block==0.9.1 +django-render-block==0.9.2 # via django-templated-email -django-reversion==5.0.0 +django-reversion==5.0.12 # via -r dependencies/pip/requirements.in -django-storages[azure,boto3]==1.13.2 +django-storages[azure,boto3]==1.14.2 # via -r dependencies/pip/requirements.in -django-taggit==2.1.0 +django-taggit==5.0.1 # via -r dependencies/pip/requirements.in -django-templated-email==3.0.0 +django-templated-email==3.0.1 # via -r dependencies/pip/requirements.in -django-timezone-field==4.2.3 +django-timezone-field==6.1.0 # via # -r dependencies/pip/requirements.in # django-celery-beat -djangorestframework==3.13.1 +djangorestframework==3.14.0 # via # -r dependencies/pip/requirements.in # djangorestframework-csv # djangorestframework-guardian # kobo-service-account -djangorestframework-csv==2.1.1 +djangorestframework-csv==3.0.2 # via -r dependencies/pip/requirements.in djangorestframework-guardian==0.3.0 # via -r dependencies/pip/requirements.in @@ -147,9 +148,9 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in -dpath==2.0.6 +dpath==2.1.6 # via -r dependencies/pip/requirements.in -ecdsa==0.17.0 +ecdsa==0.18.0 # via tlslite-ng elaphe3==0.2.0 # via -r dependencies/pip/requirements.in @@ -157,129 +158,121 @@ et-xmlfile==1.1.0 # via openpyxl gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in -idna==3.3 +idna==3.6 # via requests isodate==0.6.1 - # via msrest -jmespath==1.0.0 + # via azure-storage-blob +jmespath==1.0.1 # via # boto3 # botocore jsonfield==3.1.0 # via -r dependencies/pip/requirements.in -jwcrypto==1.0 +jwcrypto==1.5.6 # via django-oauth-toolkit -kombu==5.2.4 +kombu==5.3.5 # via celery -lxml==4.8.0 +lxml==5.1.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.3.6 +markdown==3.5.2 # via -r dependencies/pip/requirements.in +markupsafe==2.1.5 + # via werkzeug modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -msrest==0.6.21 - # via azure-storage-blob -numpy==1.22.3 +numpy==1.26.4 # via pandas -oauthlib==3.2.0 - # via - # django-oauth-toolkit - # requests-oauthlib +oauthlib==3.2.2 + # via django-oauth-toolkit openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -packaging==21.3 - # via redis -pandas==1.4.2 +pandas==2.2.1 # via -r dependencies/pip/requirements.in -pillow==9.1.0 +pillow==10.2.0 # via # -r dependencies/pip/requirements.in # elaphe3 -prompt-toolkit==3.0.29 +prompt-toolkit==3.0.43 # via click-repl -psycopg2==2.9.3 +psycopg==3.1.18 # via -r dependencies/pip/requirements.in pycparser==2.21 # via cffi -pymongo==3.12.3 +pymongo==3.13.0 # via -r dependencies/pip/requirements.in -pyparsing==3.0.8 - # via packaging -python-crontab==2.6.0 +python-crontab==3.0.0 # via django-celery-beat -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore + # celery # pandas # python-crontab -pytz==2022.1 +pytz==2024.1 # via - # celery - # django - # django-timezone-field # djangorestframework # pandas pyxform==1.9.0 # via -r dependencies/pip/requirements.in -redis==4.2.2 +redis==5.0.3 # via # -r dependencies/pip/requirements.in # celery # django-redis # django-redis-sessions # kobo-service-account -requests==2.27.1 +requests==2.31.0 # via # azure-core # django-oauth-toolkit - # msrest - # requests-oauthlib -requests-oauthlib==1.3.1 - # via msrest -s3transfer==0.5.2 +s3transfer==0.10.0 # via boto3 -sentry-sdk==1.5.10 +sentry-sdk==1.41.0 # via -r dependencies/pip/requirements.in -simplejson==3.17.6 +simplejson==3.19.2 # via -r dependencies/pip/requirements.in six==1.16.0 # via # azure-core - # click-repl - # djangorestframework-csv # ecdsa # isodate # python-dateutil -sqlparse==0.4.2 +sqlparse==0.4.4 # via django tlslite-ng==0.7.6 # via gdata-python3 -typing-extensions==4.2.0 - # via azure-core -unicodecsv==0.14.1 - # via djangorestframework-csv -urllib3==1.26.9 +typing-extensions==4.10.0 + # via + # asgiref + # azure-core + # azure-storage-blob + # jwcrypto + # psycopg +tzdata==2024.1 + # via + # celery + # django-celery-beat + # pandas +urllib3==2.0.7 # via # botocore # requests # sentry-sdk -uwsgi==2.0.21 +uwsgi==2.0.24 # via -r dependencies/pip/requirements.in -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.5 +wcwidth==0.2.13 # via prompt-toolkit -werkzeug==2.0.3 +werkzeug==3.0.1 # via -r dependencies/pip/requirements.in -wrapt==1.14.0 - # via deprecated xlrd==2.0.1 # via pyxform xlwt==1.3.0 From 0523868ac5c3f838b64a8def2c2c637d47a0d2e8 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:31:57 -0400 Subject: [PATCH 02/21] Update README --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8476f06d4..5f5caf812 100644 --- a/README.md +++ b/README.md @@ -68,22 +68,22 @@ instructions on how to install KoboToolbox. To generate a locale from scratch (ex. Spanish) ``` sh -$ django-admin.py makemessages -l es -e py,html,email,txt ; -$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin.py makemessages -d djangojs -l es && cd - ; done +$ django-admin makemessages -l es -e py,html,email,txt ; +$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin makemessages -d djangojs -l es && cd - ; done ``` To update PO files ``` sh -$ django-admin.py makemessages -a ; -$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin.py makemessages -d djangojs -a && cd - ; done +$ django-admin makemessages -a ; +$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin makemessages -d djangojs -a && cd - ; done ``` To compile MO files and update live translations ``` sh -$ django-admin.py compilemessages ; -$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin.py compilemessages && cd - ; done +$ django-admin compilemessages ; +$ for app in {main,viewer} ; do cd kobocat/apps/${app} && django-admin compilemessages && cd - ; done ``` ## Testing in KoboCAT From ea171ba895f7d4af5f421053b2093a9291aff6eb Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:32:39 -0400 Subject: [PATCH 03/21] Replace deprecated utilities with 4.2 counterparts --- onadata/apps/logger/models/instance.py | 2 +- onadata/apps/restservice/__init__.py | 3 --- onadata/apps/viewer/views.py | 2 +- onadata/libs/models/signals.py | 4 ++-- script/i18ntool.py | 8 ++++---- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index ee6ef5ad6..99f440b8d 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -10,7 +10,7 @@ from django.contrib.auth.models import User from django.contrib.gis.db import models from django.contrib.gis.geos import GeometryCollection, Point -from django.db import models as django_models, transaction +from django.db import transaction from django.db.models import Case, F, When from django.db.models.signals import post_delete from django.db.models.signals import post_save diff --git a/onadata/apps/restservice/__init__.py b/onadata/apps/restservice/__init__.py index 2a797d1f7..403813643 100644 --- a/onadata/apps/restservice/__init__.py +++ b/onadata/apps/restservice/__init__.py @@ -4,6 +4,3 @@ SERVICE_CHOICES = ( SERVICE_KPI_HOOK, ) - - -default_app_config = "onadata.apps.restservice.app.RestServiceConfig" diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index c04eb237d..48b6ddf65 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -3,6 +3,7 @@ import logging import os import re +from urllib.parse import quote as urlquote import rest_framework.request from django.conf import settings @@ -20,7 +21,6 @@ ) from django.shortcuts import get_object_or_404 from django.shortcuts import render -from django.utils.http import urlquote from django.utils.translation import gettext as t from django.views.decorators.http import require_POST from rest_framework.exceptions import AuthenticationFailed diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index aafa7feff..8bcf322ac 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -3,8 +3,8 @@ from onadata.apps.logger.models import XForm -xform_tags_add = django.dispatch.Signal(providing_args=['xform', 'tags']) -xform_tags_delete = django.dispatch.Signal(providing_args=['xform', 'tag']) +xform_tags_add = django.dispatch.Signal() +xform_tags_delete = django.dispatch.Signal() @django.dispatch.receiver(xform_tags_add, sender=XForm) diff --git a/script/i18ntool.py b/script/i18ntool.py index 002ee5f0a..179f2834d 100755 --- a/script/i18ntool.py +++ b/script/i18ntool.py @@ -110,13 +110,13 @@ def add(lang): for loc in langs: with indent(2): puts("Generating PO for %s" % loc) - shell_call("django-admin.py makemessages -l %(lang)s " + shell_call("django-admin makemessages -l %(lang)s " "-e py,html,email,txt" % {'lang': loc}) for app in I18N_APPS: with indent(4): puts("Generating PO for app %s" % app) with chdir(os.path.join(REPO_ROOT, app)): - shell_call("django-admin.py makemessages " + shell_call("django-admin makemessages " "-d djangojs -l %(lang)s" % {'lang': loc}) puts(colored.green("sucesssfuly generated %s" % loc)) @@ -154,13 +154,13 @@ def compile_mo(lang=None): for loc in langs: with indent(2): puts("Compiling %s" % loc) - shell_call("django-admin.py compilemessages -l %(lang)s " + shell_call("django-admin compilemessages -l %(lang)s " % {'lang': loc}) for app in I18N_APPS: with indent(4): puts("Compiling app %s" % app) with chdir(os.path.join(REPO_ROOT, app)): - shell_call("django-admin.py compilemessages -l %(lang)s" + shell_call("django-admin compilemessages -l %(lang)s" % {'lang': loc}) puts(colored.green("sucesssfuly compiled %s" % loc)) From d4c2bdd07110bd26346cb4f5e2b58b6d44f0472f Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:50:05 -0400 Subject: [PATCH 04/21] Remove deprecated DEFAULT_FILE_STORAGE --- onadata/settings/base.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 1f779e620..9650470eb 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -363,11 +363,23 @@ def skip_suspicious_operations(record): # If not properly overridden, leave uninitialized so Django can set the default. # (see https://docs.djangoproject.com/en/1.8/ref/settings/#default-file-storage) +default_file_storage = 'django.core.files.storage.FileSystemStorage' + if os.environ.get('KOBOCAT_DEFAULT_FILE_STORAGE'): - DEFAULT_FILE_STORAGE = env.str('KOBOCAT_DEFAULT_FILE_STORAGE') - if DEFAULT_FILE_STORAGE == 'storages.backends.s3boto3.S3Boto3Storage': + default_file_storage = env.str('KOBOCAT_DEFAULT_FILE_STORAGE') + if default_file_storage == 'storages.backends.s3boto3.S3Boto3Storage': # Force usage of custom S3 tellable Storage - DEFAULT_FILE_STORAGE = 'onadata.apps.storage_backends.s3boto3.S3Boto3Storage' + default_file_storage = 'onadata.apps.storage_backends.s3boto3.S3Boto3Storage' + + +STORAGES = { + 'default': { + 'BACKEND': default_file_storage, + }, + 'staticfiles': { + 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage', + }, +} EMAIL_BACKEND = env.str( 'EMAIL_BACKEND', 'django.core.mail.backends.filebased.EmailBackend' From 84fb72bb4c87a7b7c42692aa5f28a3deb56ce979 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:50:48 -0400 Subject: [PATCH 05/21] Replace deprecated django.utils.timezone.utc --- onadata/apps/logger/tests/models/test_instance.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 66eda9d37..68b1cecf2 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -1,10 +1,9 @@ # coding: utf-8 import os import reversion -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from dateutil import parser -from django.utils.timezone import utc from django_digest.test import DigestAuth from mock import patch @@ -30,7 +29,7 @@ def test_stores_json(self): @patch('django.utils.timezone.now') def test_json_assigns_attributes(self, mock_time): - mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) + mock_time.return_value = datetime.utcnow().replace(tzinfo=timezone.utc) self._publish_transportation_form_and_submit_instance() xform_id_string = XForm.objects.all()[0].id_string @@ -44,7 +43,7 @@ def test_json_assigns_attributes(self, mock_time): @patch('django.utils.timezone.now') def test_json_stores_user_attribute(self, mock_time): - mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) + mock_time.return_value = datetime.utcnow().replace(tzinfo=timezone.utc) self._publish_transportation_form() # submit instance with a request user From 40d743dbfc6d6ebd85daeaf553a4e23f8b430ce9 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:51:02 -0400 Subject: [PATCH 06/21] Remove unicodecsv --- onadata/libs/utils/csv_import.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 4f432e47f..63a1df42e 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -1,12 +1,12 @@ # coding: utf-8 +import csv import io import json import uuid from datetime import datetime from typing import TextIO, Union -import unicodecsv as ucsv -from django.contrib.auth.models import User +from django.core.files.uploadedfile import InMemoryUploadedFile from onadata.apps.logger.models import Instance from onadata.libs.utils.logger_tools import dict2xml, safe_create_instance @@ -70,6 +70,9 @@ def submit_csv( :py:func:`onadata.libs.utils.logger_tools.safe_create_instance` """ + if isinstance(csv_file, InMemoryUploadedFile): + csv_file = csv_file.read().decode() + if isinstance(csv_file, str): csv_file = io.StringIO(csv_file) elif csv_file is None or not hasattr(csv_file, 'read'): @@ -77,7 +80,7 @@ def submit_csv( 'Expected file or String ' 'got {} instead.'.format(type(csv_file).__name__))} - csv_reader = ucsv.DictReader(csv_file) + csv_reader = csv.DictReader(csv_file) rollback_uuids = [] submission_time = datetime.utcnow().isoformat() ona_uuid = {'formhub': {'uuid': xform.uuid}} From 913a55a4d6d77deb3869bf4a6cfbef42568a4723 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 11:56:11 -0400 Subject: [PATCH 07/21] Use XLSX - Panda dropped support for XLS --- onadata/apps/viewer/tests/test_pandas_mongo_bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py index 124e28871..9bdd10527 100644 --- a/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py +++ b/onadata/apps/viewer/tests/test_pandas_mongo_bridge.py @@ -394,7 +394,7 @@ def test_unicode_export(self): columns = ["key"] # test xls xls_df_writer = XLSDataFrameWriter(data, columns) - temp_file = NamedTemporaryFile(suffix='.xls', mode='w') + temp_file = NamedTemporaryFile(suffix='.xlsx', mode='w') excel_writer = ExcelWriter(temp_file.name) passed = False try: From 22241f06d68f478e98416cfbf0ab677ce26041f8 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 12:45:11 -0400 Subject: [PATCH 08/21] Fix csv imports --- onadata/libs/utils/csv_import.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index 63a1df42e..6274c6d98 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -70,15 +70,20 @@ def submit_csv( :py:func:`onadata.libs.utils.logger_tools.safe_create_instance` """ - if isinstance(csv_file, InMemoryUploadedFile): - csv_file = csv_file.read().decode() + + if hasattr(csv_file, 'readable'): + csv_file = io.TextIOWrapper(csv_file, encoding='utf-8') if isinstance(csv_file, str): csv_file = io.StringIO(csv_file) elif csv_file is None or not hasattr(csv_file, 'read'): - return {'error': ('Invalid param type for `csv_file`. ' - 'Expected file or String ' - 'got {} instead.'.format(type(csv_file).__name__))} + return { + 'error': ( + 'Invalid param type for `csv_file`. ' + 'Expected file or String ' + 'got {} instead.'.format(type(csv_file).__name__) + ) + } csv_reader = csv.DictReader(csv_file) rollback_uuids = [] From 2796c18245df4c72290f0064f546b36fa810c6ae Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 12:45:44 -0400 Subject: [PATCH 09/21] Remove deprecated PIL.ANTIALIAS --- onadata/libs/utils/image_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/utils/image_tools.py b/onadata/libs/utils/image_tools.py index 8215c12e5..b2a364e12 100644 --- a/onadata/libs/utils/image_tools.py +++ b/onadata/libs/utils/image_tools.py @@ -41,7 +41,7 @@ def _save_thumbnails(image, original_path, size, suffix): nm = NamedTemporaryFile(suffix='.%s' % image.format) try: # Ensure conversion to float in operations - image.thumbnail(get_dimensions(image.size, float(size)), Image.ANTIALIAS) + image.thumbnail(get_dimensions(image.size, float(size)), Image.LANCZOS) except ZeroDivisionError: pass try: From 202b65676b5e069f23c1a6cff9904e5753a1ffc7 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 12:46:56 -0400 Subject: [PATCH 10/21] Replace missed deprecated django.utils.timezone.utc --- onadata/libs/utils/logger_tools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 273893e2a..d93e254b6 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -6,7 +6,7 @@ import re import sys import traceback -from datetime import date, datetime +from datetime import date, datetime, timezone from xml.etree import ElementTree as ET from xml.parsers.expat import ExpatError try: @@ -29,7 +29,7 @@ Http404 ) from django.shortcuts import get_object_or_404 -from django.utils import timezone +from django.utils import timezone as dj_timezone from django.utils.encoding import DjangoUnicodeDecodeError, smart_str from django.utils.translation import gettext as t from kobo_service_account.utils import get_real_user @@ -647,9 +647,9 @@ def save_submission( # override date created if required if date_created_override: - if not timezone.is_aware(date_created_override): + if not dj_timezone.is_aware(date_created_override): # default to utc? - date_created_override = timezone.make_aware( + date_created_override = dj_timezone.make_aware( date_created_override, timezone.utc) instance.date_created = date_created_override instance.save() @@ -741,7 +741,7 @@ def get_soft_deleted_attachments(instance: Instance) -> list[Attachment]: | Q(media_file_basename__regex=r'^\d{10,}\.(m4a|amr)$') ) soft_deleted_attachments = list(queryset.all()) - queryset.update(deleted_at=timezone.now()) + queryset.update(deleted_at=dj_timezone.now()) return soft_deleted_attachments From 4f3787d3adcd4cfba83dccb27c335e73c805f970 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 12:51:57 -0400 Subject: [PATCH 11/21] Use pymongo update_many() instead of deprecated update() --- onadata/apps/viewer/models/parsed_instance.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 52da7ee60..2e809c1e3 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -306,10 +306,9 @@ def update_mongo(self, asynchronous=True): @staticmethod def bulk_update_validation_statuses(query, validation_status): - return xform_instances.update( + return xform_instances.update_many( query, {"$set": {VALIDATION_STATUS: validation_status}}, - multi=True, ) @staticmethod From b40926f6439905a495628a608a5144375e8a7abc Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 12:52:18 -0400 Subject: [PATCH 12/21] Remove USE_L10N var --- onadata/settings/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 9650470eb..c146ec933 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -71,10 +71,6 @@ def skip_suspicious_operations(record): # to load the internationalization machinery. USE_I18N = True -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale -USE_L10N = True - # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash. # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" From bcfc0e4d444ca8bd54880e710f2165658ddf9b1d Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 13:02:07 -0400 Subject: [PATCH 13/21] Update PyMongo --- dependencies/pip/dev_requirements.txt | 4 +++- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index ad11ba548..1ffbae159 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -154,6 +154,8 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in +dnspython==2.6.1 + # via pymongo dpath==2.1.6 # via -r dependencies/pip/requirements.in ecdsa==0.18.0 @@ -250,7 +252,7 @@ pycparser==2.21 # via cffi pygments==2.17.2 # via ipython -pymongo==3.13.0 +pymongo==4.6.2 # via -r dependencies/pip/requirements.in pytest==8.1.1 # via diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index c72fce580..6c6e698a8 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -25,7 +25,7 @@ gdata-python3 modilabs-python-utils Pillow psycopg -pymongo>=3.12,< 4.0 +pymongo lxml pyxform==1.9.0 django-reversion diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index b63d48d42..73b4046e9 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -148,6 +148,8 @@ djangorestframework-jsonp==1.0.2 # via -r dependencies/pip/requirements.in djangorestframework-xml==2.0.0 # via -r dependencies/pip/requirements.in +dnspython==2.6.1 + # via pymongo dpath==2.1.6 # via -r dependencies/pip/requirements.in ecdsa==0.18.0 @@ -202,7 +204,7 @@ psycopg==3.1.18 # via -r dependencies/pip/requirements.in pycparser==2.21 # via cffi -pymongo==3.13.0 +pymongo==4.6.2 # via -r dependencies/pip/requirements.in python-crontab==3.0.0 # via django-celery-beat From e48c556c583ef02b61c8d779d71dd48e09312446 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 12 Mar 2024 14:45:26 -0400 Subject: [PATCH 14/21] Fix CI failure --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index e357e0b76..2b5265ad3 100644 --- a/conftest.py +++ b/conftest.py @@ -58,7 +58,7 @@ def setup(request): toggle_capturing(capture_manager, stop=False) pytest.exit('User interrupted tests', pytest.ExitCode.INTERRUPTED) - if 'instances' in settings.MONGO_DB.collection_names(): + if 'instances' in settings.MONGO_DB.list_collection_names(): response = stderr_prompt( '\n\n' 'WARNING: the MongoDB collection {}.instances already exists!\n' From aa2ea3a21525bc8332b21c93e533ce63ef4b5513 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Wed, 13 Mar 2024 12:27:10 -0400 Subject: [PATCH 15/21] Freeze iPython to 8.12.3 --- dependencies/pip/dev_requirements.in | 2 +- onadata/settings/base.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dependencies/pip/dev_requirements.in b/dependencies/pip/dev_requirements.in index 51a2959c8..701a97797 100644 --- a/dependencies/pip/dev_requirements.in +++ b/dependencies/pip/dev_requirements.in @@ -1,7 +1,7 @@ -r requirements.in ipdb -ipython +ipython==8.12.3 # Max supported version by Python 3.8 shell_command sqlparse pytest diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 09aa6665f..32fad2500 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -346,8 +346,6 @@ def skip_suspicious_operations(record): SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer' -# If not properly overridden, leave uninitialized so Django can set the default. -# (see https://docs.djangoproject.com/en/1.8/ref/settings/#default-file-storage) default_file_storage = 'django.core.files.storage.FileSystemStorage' if os.environ.get('KOBOCAT_DEFAULT_FILE_STORAGE'): From be21a01e380420648c8d6c4de615643381e13c00 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 14 Mar 2024 14:37:56 -0400 Subject: [PATCH 16/21] Add Python 3.12 to CI matrix --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fc249eaba..74b79c4c0 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ['3.8', '3.10'] + python-version: ['3.8', '3.10', '3.12'] services: postgres: image: postgis/postgis:14-3.2 From 336dd3e73ddadfff965acdacd0a64399915db696 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 14 Mar 2024 15:01:07 -0400 Subject: [PATCH 17/21] Make pip dependencies compliant with Python 3.8 --- dependencies/pip/dev_requirements.txt | 42 +++++++++++++++++++-------- dependencies/pip/requirements.in | 2 +- dependencies/pip/requirements.txt | 31 +++++++++++++++----- 3 files changed, 54 insertions(+), 21 deletions(-) diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index 1ffbae159..329d6ac5f 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile dependencies/pip/dev_requirements.in @@ -30,11 +30,22 @@ azure-core==1.30.1 # django-storages azure-storage-blob==12.19.1 # via django-storages +backcall==0.2.0 + # via ipython +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" + # via + # -r dependencies/pip/requirements.in + # celery + # django + # django-celery-beat + # django-timezone-field + # kombu + # psycopg billiard==4.2.0 # via celery -boto3==1.34.60 +boto3==1.34.62 # via django-storages -botocore==1.34.60 +botocore==1.34.62 # via # boto3 # s3transfer @@ -165,9 +176,7 @@ elaphe3==0.2.0 et-xmlfile==1.1.0 # via openpyxl exceptiongroup==1.2.0 - # via - # ipython - # pytest + # via pytest executing==2.0.1 # via stack-data gdata-python3==3.0.1 @@ -176,11 +185,13 @@ httmock==1.4.0 # via -r dependencies/pip/dev_requirements.in idna==3.6 # via requests +importlib-metadata==7.0.2 + # via markdown iniconfig==2.0.0 # via pytest ipdb==0.13.13 # via -r dependencies/pip/dev_requirements.in -ipython==8.22.2 +ipython==8.12.3 # via # -r dependencies/pip/dev_requirements.in # ipdb @@ -202,7 +213,7 @@ lxml==5.1.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.5.2 +markdown==3.6 # via -r dependencies/pip/requirements.in markupsafe==2.1.5 # via werkzeug @@ -214,7 +225,7 @@ modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in mongomock==4.1.2 # via -r dependencies/pip/dev_requirements.in -numpy==1.26.4 +numpy==1.24.4 # via pandas oauthlib==3.2.2 # via django-oauth-toolkit @@ -226,12 +237,14 @@ packaging==24.0 # via # mongomock # pytest -pandas==2.2.1 +pandas==2.0.3 # via -r dependencies/pip/requirements.in parso==0.8.3 # via jedi pexpect==4.9.0 # via ipython +pickleshare==0.7.5 + # via ipython pillow==10.2.0 # via # -r dependencies/pip/requirements.in @@ -293,7 +306,7 @@ s3transfer==0.10.0 # via boto3 sentinels==1.0.0 # via mongomock -sentry-sdk==1.41.0 +sentry-sdk==1.42.0 # via -r dependencies/pip/requirements.in shell-command==0.1 # via -r dependencies/pip/dev_requirements.in @@ -328,14 +341,17 @@ typing-extensions==4.10.0 # asgiref # azure-core # azure-storage-blob + # ipython # jwcrypto + # kombu # psycopg tzdata==2024.1 # via + # backports-zoneinfo # celery # django-celery-beat # pandas -urllib3==2.0.7 +urllib3==1.26.18 # via # botocore # requests @@ -355,4 +371,6 @@ xlrd==2.0.1 # via pyxform xlwt==1.3.0 # via -r dependencies/pip/requirements.in +zipp==3.18.0 + # via importlib-metadata backports-zoneinfo==0.2.1; python_version < '3.9' diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 6c6e698a8..5103d9b6e 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -82,7 +82,7 @@ djangorestframework-guardian sentry-sdk # mimetype detection -Werkzeug # <=2.0.3 +Werkzeug # WSGI server uWSGI diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 73b4046e9..ea1b236c0 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile dependencies/pip/requirements.in @@ -28,11 +28,20 @@ azure-core==1.30.1 # django-storages azure-storage-blob==12.19.1 # via django-storages +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" + # via + # -r dependencies/pip/requirements.in + # celery + # django + # django-celery-beat + # django-timezone-field + # kombu + # psycopg billiard==4.2.0 # via celery -boto3==1.34.60 +boto3==1.34.62 # via django-storages -botocore==1.34.60 +botocore==1.34.62 # via # boto3 # s3transfer @@ -162,6 +171,8 @@ gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in idna==3.6 # via requests +importlib-metadata==7.0.2 + # via markdown isodate==0.6.1 # via azure-storage-blob jmespath==1.0.1 @@ -178,13 +189,13 @@ lxml==5.1.0 # via # -r dependencies/pip/requirements.in # gdata-python3 -markdown==3.5.2 +markdown==3.6 # via -r dependencies/pip/requirements.in markupsafe==2.1.5 # via werkzeug modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in -numpy==1.26.4 +numpy==1.24.4 # via pandas oauthlib==3.2.2 # via django-oauth-toolkit @@ -192,7 +203,7 @@ openpyxl==3.0.9 # via # -r dependencies/pip/requirements.in # pyxform -pandas==2.2.1 +pandas==2.0.3 # via -r dependencies/pip/requirements.in pillow==10.2.0 # via @@ -233,7 +244,7 @@ requests==2.31.0 # django-oauth-toolkit s3transfer==0.10.0 # via boto3 -sentry-sdk==1.41.0 +sentry-sdk==1.42.0 # via -r dependencies/pip/requirements.in simplejson==3.19.2 # via -r dependencies/pip/requirements.in @@ -253,13 +264,15 @@ typing-extensions==4.10.0 # azure-core # azure-storage-blob # jwcrypto + # kombu # psycopg tzdata==2024.1 # via + # backports-zoneinfo # celery # django-celery-beat # pandas -urllib3==2.0.7 +urllib3==1.26.18 # via # botocore # requests @@ -279,4 +292,6 @@ xlrd==2.0.1 # via pyxform xlwt==1.3.0 # via -r dependencies/pip/requirements.in +zipp==3.18.0 + # via importlib-metadata backports-zoneinfo==0.2.1; python_version < '3.9' From 0edc30315ace730ca982014f7c6c94e5f11db804 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 28 Mar 2024 10:32:54 -0400 Subject: [PATCH 18/21] Use global settings and update them when storage differs --- onadata/settings/base.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 32fad2500..c505608b7 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -8,6 +8,7 @@ import environ from celery.schedules import crontab +from django.conf import global_settings from django.core.exceptions import SuspiciousOperation from pymongo import MongoClient @@ -344,25 +345,17 @@ def skip_suspicious_operations(record): if SESSION_COOKIE_DOMAIN: SESSION_COOKIE_NAME = env.str('SESSION_COOKIE_NAME', 'kobonaut') -SESSION_SERIALIZER = 'django.contrib.sessions.serializers.JSONSerializer' - -default_file_storage = 'django.core.files.storage.FileSystemStorage' +# Storage configuration +STORAGES = global_settings.STORAGES if os.environ.get('KOBOCAT_DEFAULT_FILE_STORAGE'): - default_file_storage = env.str('KOBOCAT_DEFAULT_FILE_STORAGE') - if default_file_storage == 'storages.backends.s3boto3.S3Boto3Storage': + STORAGES['default']['BACKEND'] = env.str( + 'KOBOCAT_DEFAULT_FILE_STORAGE' + ) + if STORAGES['default']['BACKEND'] == 'storages.backends.s3boto3.S3Boto3Storage': # Force usage of custom S3 tellable Storage - default_file_storage = 'onadata.apps.storage_backends.s3boto3.S3Boto3Storage' - + STORAGES['default']['BACKEND'] = 'onadata.apps.storage_backends.s3boto3.S3Boto3Storage' -STORAGES = { - 'default': { - 'BACKEND': default_file_storage, - }, - 'staticfiles': { - 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage', - }, -} EMAIL_BACKEND = env.str( 'EMAIL_BACKEND', 'django.core.mail.backends.filebased.EmailBackend' From 7ab95291c3112c2f86a9a51f2d2f48d58d2f609a Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 2 Apr 2024 10:22:57 -0400 Subject: [PATCH 19/21] Get rid of deprecated "get_storage_class" --- .../commands/change_s3_media_permissions.py | 30 +++++++++----- .../management/commands/move_media_to_s3.py | 40 ++++++++++--------- onadata/settings/base.py | 5 ++- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 57d53c902..93cbde4f2 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -4,36 +4,46 @@ import sys from django.core.management.base import BaseCommand, CommandError -from django.core.files.storage import get_storage_class +from django.core.files.storage import ( + default_storage, + storages, + InvalidStorageError, +) class Command(BaseCommand): - help = "Makes all s3 files private" + help = 'Makes all s3 files private' def handle(self, *args, **kwargs): permissions = ('private', 'public-read', 'authenticated-read') if len(args) < 1: - raise CommandError("Missing permission argument") + raise CommandError('Missing permission argument') permission = args[0] if permission not in permissions: raise CommandError( - "Expected %s as permission" % ' or '.join(permissions)) + "Expected %s as permission" % ' or '.join(permissions) + ) try: - s3 = get_storage_class('storages.backends.s3boto3.S3Boto3Storage')() - except: - print("Missing necessary libraries. Try running: pip install " - "-r requirements-s3.pip") + # If `local` is present in `settings.STORAGES`, S3 is well configured + fs = storages['local'] + s3 = default_storage + except InvalidStorageError: + print( + 'You must first set `KOBOCAT_DEFAULT_FILE_STORAGE` env variable ' + 'to `storages.backends.s3boto3.S3Boto3Storage`' + ) sys.exit(1) + else: all_files = s3.bucket.list() for i, f in enumerate(all_files): f.set_acl(permission) if i % 1000 == 0: - print(i, "file objects processed") + print(i, 'file objects processed') - print("a total of", i, "file objects processed") + print('a total of', i, 'file objects processed') diff --git a/onadata/apps/logger/management/commands/move_media_to_s3.py b/onadata/apps/logger/management/commands/move_media_to_s3.py index 14ec12340..ff7c2bf84 100644 --- a/onadata/apps/logger/management/commands/move_media_to_s3.py +++ b/onadata/apps/logger/management/commands/move_media_to_s3.py @@ -3,33 +3,35 @@ # coding: utf-8 import sys -from django.core.files.storage import default_storage, get_storage_class +from django.core.files.storage import ( + default_storage, + storages, + InvalidStorageError, +) from django.core.management.base import BaseCommand from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.attachment import upload_to as\ - attachment_upload_to -from onadata.apps.logger.models.xform import XForm, upload_to as\ - xform_upload_to +from onadata.apps.logger.models.attachment import ( + upload_to as attachment_upload_to, +) +from onadata.apps.logger.models.xform import XForm, upload_to as xform_upload_to class Command(BaseCommand): - help = ("Moves all attachments and xls files " - "to s3 from the local file system storage.") + help = ( + 'Moves all attachments and xls files ' + 'to s3 from the local file system storage.' + ) def handle(self, *args, **kwargs): try: - fs = get_storage_class( - 'django.core.files.storage.FileSystemStorage')() - s3 = get_storage_class('storages.backends.s3boto3.S3Boto3Storage')() - except: - print("Missing necessary libraries. Try running: pip install -r" - "requirements/s3.pip") - sys.exit(1) - - if default_storage.__class__ != s3.__class__: - print("You must first set your default storage to s3 in your " - "local_settings.py file.") + fs = storages['local'] + s3 = default_storage + except InvalidStorageError: + print( + 'You must first set `KOBOCAT_DEFAULT_FILE_STORAGE` env variable ' + 'to `storages.backends.s3boto3.S3Boto3Storage`' + ) sys.exit(1) classes_to_move = [ @@ -45,7 +47,7 @@ def handle(self, *args, **kwargs): if f.name and fs.exists(f.name) and not s3.exists( upload_to(i, f.name)): f.save(fs.path(f.name), fs.open(fs.path(f.name))) - print ("\t+ '%(fname)s'\n\t---> '%(url)s'" + print("\t+ '%(fname)s'\n\t---> '%(url)s'" % {'fname': fs.path(old_filename), 'url': f.url}) else: print("\t- (f.name=%s, fs.exists(f.name)=%s, not s3.exist" diff --git a/onadata/settings/base.py b/onadata/settings/base.py index c505608b7..1480214a7 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -355,7 +355,10 @@ def skip_suspicious_operations(record): if STORAGES['default']['BACKEND'] == 'storages.backends.s3boto3.S3Boto3Storage': # Force usage of custom S3 tellable Storage STORAGES['default']['BACKEND'] = 'onadata.apps.storage_backends.s3boto3.S3Boto3Storage' - + # needed for some management commands to move files from local storage to S3 + STORAGES['local'] = { + 'BACKEND': 'django.core.files.storage.FileSystemStorage', + } EMAIL_BACKEND = env.str( 'EMAIL_BACKEND', 'django.core.mail.backends.filebased.EmailBackend' From 0bb0357bfb34fa5966c98c88da9cab9b6850cff4 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 11 Apr 2024 08:15:21 -0400 Subject: [PATCH 20/21] Add WebP to allowed content types --- onadata/settings/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/onadata/settings/base.py b/onadata/settings/base.py index 1480214a7..d805542df 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -574,6 +574,7 @@ def skip_suspicious_operations(record): 'image/jpeg', 'image/png', 'image/svg+xml', + 'image/webp', 'video/3gpp', 'video/mp4', 'video/quicktime', From cec4da1a7f5b1beb37ba7c921fb181cbb1700a06 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Thu, 11 Apr 2024 12:40:37 -0400 Subject: [PATCH 21/21] Move signals to their own files --- onadata/apps/api/viewsets/data_viewset.py | 14 ++++----- onadata/apps/logger/__init__.py | 16 ---------- onadata/apps/logger/app.py | 16 ++++++++++ onadata/apps/main/app.py | 10 +++++++ onadata/apps/main/models/user_profile.py | 27 ----------------- onadata/apps/main/signals.py | 26 ++++++++++++++-- onadata/apps/restservice/app.py | 7 +++-- onadata/apps/viewer/app.py | 10 +++++++ onadata/apps/viewer/models/data_dictionary.py | 16 +++------- onadata/apps/viewer/models/export.py | 10 ------- onadata/apps/viewer/models/parsed_instance.py | 10 +------ onadata/apps/viewer/signals.py | 30 +++++++++++++++++++ onadata/settings/base.py | 8 ++--- 13 files changed, 109 insertions(+), 91 deletions(-) create mode 100644 onadata/apps/logger/app.py create mode 100644 onadata/apps/main/app.py create mode 100644 onadata/apps/viewer/app.py create mode 100644 onadata/apps/viewer/signals.py diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index cbf97a69d..970b839cb 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -29,14 +29,12 @@ nullify_exports_time_of_last_submission, update_xform_submission_count_delete, ) -from onadata.apps.main.models import UserProfile -from onadata.apps.viewer.models.parsed_instance import ( - _remove_from_mongo, - ParsedInstance, -) +from onadata.apps.viewer.models.parsed_instance import ParsedInstance +from onadata.apps.viewer.signals import remove_from_mongo from onadata.libs.renderers import renderers from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) + AnonymousUserPublicFormsMixin, +) from onadata.apps.api.permissions import ( EnketoSubmissionEditPermissions, EnketoSubmissionViewPermissions, @@ -413,7 +411,7 @@ def bulk_delete(self, request, *args, **kwargs): postgres_query, mongo_query = self.__build_db_queries(xform, request.data) # Disconnect signals to speed-up bulk deletion - pre_delete.disconnect(_remove_from_mongo, sender=ParsedInstance) + pre_delete.disconnect(remove_from_mongo, sender=ParsedInstance) post_delete.disconnect( nullify_exports_time_of_last_submission, sender=Instance, dispatch_uid='nullify_exports_time_of_last_submission', @@ -445,7 +443,7 @@ def bulk_delete(self, request, *args, **kwargs): ) finally: # Pre_delete signal needs to be re-enabled for parsed instance - pre_delete.connect(_remove_from_mongo, sender=ParsedInstance) + pre_delete.connect(remove_from_mongo, sender=ParsedInstance) post_delete.connect( nullify_exports_time_of_last_submission, sender=Instance, diff --git a/onadata/apps/logger/__init__.py b/onadata/apps/logger/__init__.py index db7dc9515..e69de29bb 100644 --- a/onadata/apps/logger/__init__.py +++ b/onadata/apps/logger/__init__.py @@ -1,16 +0,0 @@ -# coding: utf-8 -from django.apps import AppConfig - - -class LoggerAppConfig(AppConfig): - - name = 'onadata.apps.logger' - - def ready(self): - # Makes sure all signal handlers are connected - from onadata.apps.logger import signals - # Monkey patch reversion package to insert real user in DB instead of - # system account superuser. - from kobo_service_account.utils import reversion_monkey_patch - reversion_monkey_patch() - super().ready() diff --git a/onadata/apps/logger/app.py b/onadata/apps/logger/app.py new file mode 100644 index 000000000..db7dc9515 --- /dev/null +++ b/onadata/apps/logger/app.py @@ -0,0 +1,16 @@ +# coding: utf-8 +from django.apps import AppConfig + + +class LoggerAppConfig(AppConfig): + + name = 'onadata.apps.logger' + + def ready(self): + # Makes sure all signal handlers are connected + from onadata.apps.logger import signals + # Monkey patch reversion package to insert real user in DB instead of + # system account superuser. + from kobo_service_account.utils import reversion_monkey_patch + reversion_monkey_patch() + super().ready() diff --git a/onadata/apps/main/app.py b/onadata/apps/main/app.py new file mode 100644 index 000000000..2e12e15d1 --- /dev/null +++ b/onadata/apps/main/app.py @@ -0,0 +1,10 @@ +# coding: utf-8 +from django.apps import AppConfig + + +class MainConfig(AppConfig): + name = 'onadata.apps.main' + + def ready(self): + from onadata.apps.main import signals + super().ready() diff --git a/onadata/apps/main/models/user_profile.py b/onadata/apps/main/models/user_profile.py index 83e742f01..e70c9892f 100644 --- a/onadata/apps/main/models/user_profile.py +++ b/onadata/apps/main/models/user_profile.py @@ -2,15 +2,9 @@ from django.conf import settings from django.contrib.auth.models import User from django.db import models -from django.db.models.signals import post_save -from django.utils import timezone -from django.utils.translation import gettext_lazy from guardian.conf import settings as guardian_settings -from guardian.shortcuts import get_perms_for_model, assign_perm -from rest_framework.authtoken.models import Token from onadata.apps.logger.fields import LazyDefaultBooleanField -from onadata.apps.main.signals import set_api_permissions from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.gravatar import get_gravatar_img_link, gravatar_exists @@ -63,27 +57,6 @@ class Meta: ) -def create_auth_token(sender, instance=None, created=False, **kwargs): - if created: - Token.objects.create(user=instance) - - -post_save.connect(create_auth_token, sender=User, dispatch_uid='auth_token') - -post_save.connect(set_api_permissions, sender=User, - dispatch_uid='set_api_permissions') - - -def set_object_permissions(sender, instance=None, created=False, **kwargs): - if created: - for perm in get_perms_for_model(UserProfile): - assign_perm(perm.codename, instance.user, instance) - - -post_save.connect(set_object_permissions, sender=UserProfile, - dispatch_uid='set_object_permissions') - - def get_anonymous_user_instance(User): """ Force `AnonymousUser` to be saved with `pk` == `ANONYMOUS_USER_ID` diff --git a/onadata/apps/main/signals.py b/onadata/apps/main/signals.py index e8c07df4e..e0776576f 100644 --- a/onadata/apps/main/signals.py +++ b/onadata/apps/main/signals.py @@ -1,5 +1,27 @@ -# coding: utf-8 +from django.db.models.signals import post_save +from django.dispatch import receiver +from guardian.shortcuts import assign_perm, get_perms_for_model +from rest_framework.authtoken.models import Token + +from django.contrib.auth.models import User +from onadata.apps.main.models.user_profile import UserProfile +from onadata.libs.utils.user_auth import set_api_permissions_for_user + + +@receiver(post_save, sender=User, dispatch_uid='set_api_permissions') def set_api_permissions(sender, instance=None, created=False, **kwargs): - from onadata.libs.utils.user_auth import set_api_permissions_for_user if created: set_api_permissions_for_user(instance) + + +@receiver(post_save, sender=User) +def create_auth_token(sender, instance=None, created=False, **kwargs): + if created: + Token.objects.create(user=instance) + + +@receiver(post_save, sender=UserProfile, dispatch_uid='set_object_permissions') +def set_object_permissions(sender, instance=None, created=False, **kwargs): + if created: + for perm in get_perms_for_model(UserProfile): + assign_perm(perm.codename, instance.user, instance) diff --git a/onadata/apps/restservice/app.py b/onadata/apps/restservice/app.py index 382f15eb5..e63841f5d 100644 --- a/onadata/apps/restservice/app.py +++ b/onadata/apps/restservice/app.py @@ -3,9 +3,10 @@ class RestServiceConfig(AppConfig): - name = "onadata.apps.restservice" - verbose_name = "restservice" + name = 'onadata.apps.restservice' + verbose_name = 'restservice' def ready(self): # Register RestService signals - import onadata.apps.restservice.signals + from onadata.apps.restservice import signals + super().ready() diff --git a/onadata/apps/viewer/app.py b/onadata/apps/viewer/app.py new file mode 100644 index 000000000..a18be4ca0 --- /dev/null +++ b/onadata/apps/viewer/app.py @@ -0,0 +1,10 @@ +# coding: utf-8 +from django.apps import AppConfig + + +class ViewerConfig(AppConfig): + name = 'onadata.apps.viewer' + + def ready(self): + from onadata.apps.viewer import signals + super().ready() diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index b2f429366..4b88273fe 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -4,9 +4,7 @@ from xml.dom import Node from django.db import models -from django.db.models.signals import post_save from django.utils.encoding import smart_str -from guardian.shortcuts import assign_perm, get_perms_for_model from pyxform import SurveyElementBuilder from pyxform.builder import create_survey_from_xls from pyxform.question import Question @@ -17,8 +15,10 @@ from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml from onadata.apps.api.mongo_helper import MongoHelper from onadata.libs.utils.common_tags import UUID, SUBMISSION_TIME, TAGS, NOTES -from onadata.libs.utils.export_tools import question_types_to_exclude,\ - DictOrganizer +from onadata.libs.utils.export_tools import ( + question_types_to_exclude, + DictOrganizer, +) from onadata.libs.utils.model_tools import queryset_iterator, set_uuid @@ -425,11 +425,3 @@ def _mark_start_time_boolean(self): def get_survey_elements_of_type(self, element_type): return [e for e in self.get_survey_elements() if e.type == element_type] - - -def set_object_permissions(sender, instance=None, created=False, **kwargs): - if created: - for perm in get_perms_for_model(XForm): - assign_perm(perm.codename, instance.user, instance) -post_save.connect(set_object_permissions, sender=DataDictionary, - dispatch_uid='xform_object_permissions') diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index f643a5c7f..b50d5dd78 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -4,18 +4,11 @@ from django.core.files.storage import default_storage from django.db import models -from django.db.models.signals import post_delete from django.utils.translation import gettext as t from onadata.apps.logger.models import XForm -def export_delete_callback(sender, **kwargs): - export = kwargs['instance'] - if export.filepath and default_storage.exists(export.filepath): - default_storage.delete(export.filepath) - - class Export(models.Model): class ExportTypeError(Exception): def __str__(self): @@ -177,6 +170,3 @@ def exports_outdated(cls, xform, export_type): def is_filename_unique(cls, xform, filename): return Export.objects.filter( xform=xform, filename=filename).count() == 0 - - -post_delete.connect(export_delete_callback, sender=Export) diff --git a/onadata/apps/viewer/models/parsed_instance.py b/onadata/apps/viewer/models/parsed_instance.py index 2e809c1e3..7ffb59cdc 100644 --- a/onadata/apps/viewer/models/parsed_instance.py +++ b/onadata/apps/viewer/models/parsed_instance.py @@ -5,7 +5,7 @@ from dateutil import parser from django.conf import settings from django.db import models -from django.db.models.signals import pre_delete + from django.utils.translation import gettext as t from pymongo.errors import PyMongoError @@ -411,11 +411,3 @@ def _get_attachments_from_instance(instance): attachments.append(attachment) return attachments - - -def _remove_from_mongo(sender, **kwargs): - instance_id = kwargs.get('instance').instance.id - xform_instances.delete_one({'_id': instance_id}) - - -pre_delete.connect(_remove_from_mongo, sender=ParsedInstance) diff --git a/onadata/apps/viewer/signals.py b/onadata/apps/viewer/signals.py new file mode 100644 index 000000000..b2a316052 --- /dev/null +++ b/onadata/apps/viewer/signals.py @@ -0,0 +1,30 @@ +from django.conf import settings +from django.core.files.storage import default_storage +from django.db.models.signals import post_delete, post_save, pre_delete +from django.dispatch import receiver +from guardian.shortcuts import assign_perm, get_perms_for_model + +from onadata.apps.logger.models import XForm +from onadata.apps.viewer.models.data_dictionary import DataDictionary +from onadata.apps.viewer.models.export import Export +from onadata.apps.viewer.models.parsed_instance import ParsedInstance + + +@receiver(post_delete, sender=Export) +def export_delete_callback(sender, **kwargs): + export = kwargs['instance'] + if export.filepath and default_storage.exists(export.filepath): + default_storage.delete(export.filepath) + + +@receiver(post_save, sender=DataDictionary, dispatch_uid='xform_object_permissions') +def set_object_permissions(sender, instance=None, created=False, **kwargs): + if created: + for perm in get_perms_for_model(XForm): + assign_perm(perm.codename, instance.user, instance) + + +@receiver(pre_delete, sender=ParsedInstance) +def remove_from_mongo(sender, **kwargs): + instance_id = kwargs.get('instance').instance.id + settings.MONGO_DB.instances.delete_one({'_id': instance_id}) diff --git a/onadata/settings/base.py b/onadata/settings/base.py index d805542df..9a9cd242b 100644 --- a/onadata/settings/base.py +++ b/onadata/settings/base.py @@ -197,14 +197,14 @@ def skip_suspicious_operations(record): 'django_digest', 'corsheaders', 'oauth2_provider', - 'onadata.apps.logger.LoggerAppConfig', + 'onadata.apps.logger.app.LoggerAppConfig', 'rest_framework', 'rest_framework.authtoken', 'taggit', 'readonly', - 'onadata.apps.viewer', - 'onadata.apps.main', - 'onadata.apps.restservice', + 'onadata.apps.viewer.app.ViewerConfig', + 'onadata.apps.main.app.MainConfig', + 'onadata.apps.restservice.app.RestServiceConfig', 'onadata.apps.api', 'guardian', 'onadata.libs',