From 8955e0ca99a7defe1478c88917217de24259741d Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Fri, 16 Apr 2021 13:30:50 +0200 Subject: [PATCH 1/9] implement plugin data export --- src/mortimer/templates/export.html | 119 ++++++++++++++++--------- src/mortimer/utils.py | 28 +++++- src/mortimer/web_experiments/routes.py | 81 ++++++++++++++++- 3 files changed, 181 insertions(+), 47 deletions(-) diff --git a/src/mortimer/templates/export.html b/src/mortimer/templates/export.html index 9f7ad99..207db5b 100644 --- a/src/mortimer/templates/export.html +++ b/src/mortimer/templates/export.html @@ -35,13 +35,15 @@

Select Experiment Version

Main experiment data

- Contains the values that are saved by input elements in an experiment and the additional data dictionary. + Contains the values that are saved by input elements in an experiment and the additional data + dictionary.
- +
@@ -60,8 +62,10 @@

Codebook

- - + +
@@ -69,66 +73,95 @@

Codebook

-
-
-
-

Move history

- - Information on participant's movements in the experiment. Each row contains information - about one move, i.e. there can be lots of rows for a single participant. - -
-
+
+
+
+

Move history

+ + Information on participant's movements in the experiment. Each row contains information + about one move, i.e. there can be lots of rows for a single participant. + +
+
- - + + +
-
-
-
-
-

Full experiment data

- - Main experiment data, element information (codebook) and values, and the movement history in JSON - format. Unlinked data is not included here. - -
-
+
+
+
+

Full experiment data

+ + Main experiment data, element information (codebook) and values, and the movement history in + JSON + format. Unlinked data is not included here. + +
+
- + +
-
-
-
-
-
-

Unlinked experiment data

- - Shuffled unlinked data: Values that are saved on UnlinkedDataPages. Contains no data that allows linking with specific experiment sessions. - +
+
+
+
+

Unlinked experiment data

+ + Shuffled unlinked data: Values that are saved on UnlinkedDataPages. Contains no + data that allows linking with specific experiment sessions. + +
+
+ + + + + +
-
+
+
- - - +
+
+
+
+

Plugin data

+ + Data provided by plugins. + +
+
+ + + + +
-
-{% endblock content %} \ No newline at end of file +{% endblock content %} diff --git a/src/mortimer/utils.py b/src/mortimer/utils.py index 6158921..c98d2a0 100644 --- a/src/mortimer/utils.py +++ b/src/mortimer/utils.py @@ -3,6 +3,7 @@ import json, os, re, subprocess from datetime import datetime from uuid import uuid4 +from typing import Iterator from cryptography.fernet import Fernet from flask import current_app, render_template, url_for @@ -14,6 +15,24 @@ from mortimer import mail +def get_plugin_data_queries(exp) -> Iterator[dict]: + db = get_user_collection() + f = {"exp_id": str(exp.id), "exp_plugin_queries": {"$exists": True, "$ne": {}}} + cursor = db.find(f, projection={"_id": False, "exp_plugin_queries": True}) + + out = [] + for doc in cursor: + query_list = doc.get("exp_plugin_queries", None) + if not query_list: + continue + for q in query_list: + if not q: + continue + if not q in out: + out.append(q) + yield q + + def sanitize_db_cred(): dbauth = copy.copy(current_app.config["MONGODB_SETTINGS"]) @@ -37,7 +56,8 @@ def get_alfred_db(): def get_user_collection(): """Return a users alfred collection. - For this function to work, the DB User specified in mortimers configuration needs to have the right access privileges for the given database. + For this function to work, the DB User specified in mortimers + configuration needs to have the right access privileges for the given database. :return: Collection object. :rtype: pymongo.collection.Collection @@ -49,6 +69,12 @@ def get_user_collection(): return db[colname] +def get_user_misc_collection(): + db = get_alfred_db() + colname = current_user.alfred_col_misc + return db[colname] + + def create_fernet(): """Create a fernet instance for encryption, using mortimers secret key.""" diff --git a/src/mortimer/web_experiments/routes.py b/src/mortimer/web_experiments/routes.py index 93fc0fa..351d59b 100644 --- a/src/mortimer/web_experiments/routes.py +++ b/src/mortimer/web_experiments/routes.py @@ -28,7 +28,8 @@ request, send_file, url_for, - make_response + make_response, + session ) from flask_login import current_user, login_required from werkzeug.utils import secure_filename @@ -59,7 +60,8 @@ display_directory, get_user_collection, get_alfred_db, - create_fernet + create_fernet, + get_plugin_data_queries ) web_experiments = Blueprint("web_experiments", __name__) @@ -620,8 +622,23 @@ def export(username, experiment_title): elif dtype == "full": return redirect(url_for("web_experiments.export_full_data", experiment_title=experiment.title, username=experiment.author, versions=versions)) + + elif dtype == "plugin": + plugin_data_type = request.values.get("plugin_export_select") + if not plugin_data_type: + flash("No data found for your search.", "info") + return redirect(url_for("web_experiments.export", experiment_title=experiment.title, username=experiment.author)) + queries = get_plugin_data_queries(exp=experiment) + plugin_query = next((q for q in queries if q["type"] == plugin_data_type), None) + session["plugin_data_query"] = plugin_query + return redirect(url_for("web_experiments.export_plugin_data", experiment_title=experiment.title, username=experiment.author, versions=versions)) + + queries = get_plugin_data_queries(exp=experiment) + query_tuples = [] + for q in queries: + query_tuples.append((q["title"], q["type"])) - return render_template("export.html", experiment=experiment) + return render_template("export.html", experiment=experiment, plugin_queries=query_tuples) @web_experiments.route("///export_main_data//") @@ -921,6 +938,64 @@ def export_full_data(username, experiment_title, versions: str): ) +@web_experiments.route("///export_plugin_data/") +@login_required +def export_plugin_data(username, experiment_title, versions: str): + experiment = WebExperiment.objects.get_or_404( # pylint: disable=no-member + title=experiment_title, author=username + ) + if experiment.author != current_user.username: + abort(403) + + query = session["plugin_data_query"]["query"] + + db = get_alfred_db() + col = current_user.alfred_col_misc + + if not db[col].find_one(): + flash("No data found.", "info") + return redirect( + url_for( + "web_experiments.export", + username=experiment.author, + experiment_title=experiment.title, + ) + ) + + versions = versions.split("$VERSIONSEP$") + if not "all" in versions: + query["filter"].update({"exp_version": {"$in": versions}}) + + data = db[col].find(**query) + + if not data: + flash("No data found for your search", "info") + return redirect(url_for("web_experiments.export", experiment_title=experiment.title, username=experiment.author)) + + dlist = list(data) + + for doc in dlist: # turn ObjectID into string to make it json serializable + doc["_id"] = str(doc["_id"]) + + # decrypt if necessary + if session["plugin_data_query"].get("encrypted", False): + fern = create_fernet() + key = fern.decrypt(current_user.encryption_key) + dlist = data_manager.decrypt_recursively(dlist, key=key) + + f = json.dumps(dlist, indent=4, sort_keys=True) + + filename = session["plugin_data_query"]["type"] + fn = f"{filename}.json" + return send_file( + make_str_bytes(f), + mimetype="application/json", + as_attachment=True, + attachment_filename=fn, + cache_timeout=1, + ) + + @web_experiments.route("///data", methods=["GET"]) @login_required def data(username, experiment_title): From 488ccd44b48e130570d6e31d9c699d675c8fb47d Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Fri, 16 Apr 2021 13:31:15 +0200 Subject: [PATCH 2/9] Delete web_export.html --- src/mortimer/templates/web_export.html | 247 ------------------------- 1 file changed, 247 deletions(-) delete mode 100644 src/mortimer/templates/web_export.html diff --git a/src/mortimer/templates/web_export.html b/src/mortimer/templates/web_export.html deleted file mode 100644 index ac94ef3..0000000 --- a/src/mortimer/templates/web_export.html +++ /dev/null @@ -1,247 +0,0 @@ -{% extends "layout_experiment.html" %} - -{% block content %} - - -
- - {# Experiment Data #} -
-
-
-

Experiment Data

-
- -
- {{ form_exp_data.hidden_tag() }} - - -
- -
-
- {{ form_exp_data.data_type.label }} -
-
-
- {% for subfield in form_exp_data.data_type %} - - {% if subfield.checked %} -
- -
- {% for error in form_exp_data.data_type.errors %} - {{ error }} - {% endfor %} -
- - - Unlinked data is data that was collected on an UnlinkedDataPage. It is - saved separately from experiment data and cannot be linked to a specific experiment - session. As such, unlinked data can be suitable for collecting personalized data. - -
-
- -
-
- {{ form_exp_data.file_type.label }} -
- -
-
- {% for subfield in form_exp_data.file_type %} - {% if subfield.checked %} -
- -
- {% for error in form_exp_data.file_type.errors %} - {{ error }} - {% endfor %} -
- - Semicolon-separated .csv is recommended for working with - Excel. - -
- -
- -
-
- {{ form_exp_data.version.label(class="form-control-label") }} -
- -
- {% if form_exp_data.version.errors %} - {{ form_exp_data.version(class="form-control is-invalid") }} - -
- {% for error in form_exp_data.version.errors %} - {{ error }} - {% endfor %} -
- - {% else %} - {{ form_exp_data.version(class="form-control") }} - {% endif %} - - - By pressing command (Mac) or Strg (Windows) while clicking, - you can - select multiple versions at once. If you select "all versions", all data will be - exported, no - matter which other versions you select. - -
-
- -
-
- -
-
- -
-
- -
- -
- -
-
- - - - - {# Codebook #} -
- -
-
-

Codebook

-
- -
- {{ form_codebook.hidden_tag() }} - - -
-
- {{ form_codebook.file_type.label }} -
- -
-
- {% for subfield in form_codebook.file_type %} - {% if subfield.checked %} -
- -
- {% for error in form_codebook.file_type.errors %} - {{ error }} - {% endfor %} -
-
- -
- - -
-
- {{ form_codebook.version.label(class="form-control-label") }} -
- -
- {% if form_codebook.version.errors %} - {{ form_codebook.version(class="form-control is-invalid") }} - -
- {% for error in form_codebook.version.errors %} - {{ error }} - {% endfor %} -
- - {% else %} - {{ form_codebook.version(class="form-control") }} - {% endif %} - -
-
- - - -
-
- -
-
- -
-
- -
-
- -
-
- Export - Base - Codebook - - The base codebook describes those variables in an alfred dataset that are not linked to - specific - input elements. They are associated with pages, sections, or the whole experiment. - -
-
- - - -
- - -
-
- -
- - - - - - -{% endblock content %} \ No newline at end of file From 766c918f68530850754e5323a47269a78078186a Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Sat, 17 Apr 2021 10:04:54 +0200 Subject: [PATCH 3/9] Update alfredo.py --- src/mortimer/web_experiments/alfredo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mortimer/web_experiments/alfredo.py b/src/mortimer/web_experiments/alfredo.py index da5eb68..c50312c 100644 --- a/src/mortimer/web_experiments/alfredo.py +++ b/src/mortimer/web_experiments/alfredo.py @@ -209,7 +209,7 @@ def start(expid): abort(500) try: - exp_session.start() + exp_session._start() experiment_manager.save(sid, exp_session) except Exception: msg = "An exception occured during experiment startup." @@ -256,7 +256,7 @@ def experiment(): data.pop("directjump", None) data.pop("par", None) - experiment.movement_manager.current_page.set_data(data) + experiment.movement_manager.current_page._set_data(data) if move is None and not data: pass elif move: From 4590c7519280b9f1a8f9bb86f668eef09095c6ed Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Sat, 17 Apr 2021 10:05:27 +0200 Subject: [PATCH 4/9] Update _version.py --- src/mortimer/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mortimer/_version.py b/src/mortimer/_version.py index 2fc032c..3e912dd 100644 --- a/src/mortimer/_version.py +++ b/src/mortimer/_version.py @@ -3,4 +3,4 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = "0.8.4" +__version__ = "0.8.5" From 70c3733eb57fc2b002d20d13c064567edcb0f7cb Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Sat, 17 Apr 2021 12:27:31 +0200 Subject: [PATCH 5/9] update dependencies --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e3b1cbb..7bb9b28 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,8 @@ }, package_dir={"": "src"}, install_requires=[ - "alfred3>=1.1.4", - "cryptography>=2.9", + "alfred3>=2.0", + "cryptography>=3.4", "email_validator>=1.1", "flask>=1.1.2", "flask_bcrypt>=0.7.1", From 93c674cbfde9f554e4051add0856283440c269d7 Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Mon, 19 Apr 2021 09:34:57 +0200 Subject: [PATCH 6/9] Update _version.py --- src/mortimer/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mortimer/_version.py b/src/mortimer/_version.py index 3e912dd..f044b88 100644 --- a/src/mortimer/_version.py +++ b/src/mortimer/_version.py @@ -3,4 +3,4 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = "0.8.5" +__version__ = "0.8.5b1" From cdb96d9f32d0154e3b439df4e97e15ac22d80fe0 Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Mon, 19 Apr 2021 14:48:16 +0200 Subject: [PATCH 7/9] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7bb9b28..6c3f7bf 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ }, package_dir={"": "src"}, install_requires=[ - "alfred3>=2.0", + "alfred3>=2.0.0b13", "cryptography>=3.4", "email_validator>=1.1", "flask>=1.1.2", From c2e8127203d9dea9c660101e4f941b972503d4b8 Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Tue, 20 Apr 2021 08:32:19 +0200 Subject: [PATCH 8/9] update version --- setup.py | 2 +- src/mortimer/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6c3f7bf..4c4ee42 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ }, package_dir={"": "src"}, install_requires=[ - "alfred3>=2.0.0b13", + "alfred3>=2.0.0", "cryptography>=3.4", "email_validator>=1.1", "flask>=1.1.2", diff --git a/src/mortimer/_version.py b/src/mortimer/_version.py index f044b88..3e912dd 100644 --- a/src/mortimer/_version.py +++ b/src/mortimer/_version.py @@ -3,4 +3,4 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = "0.8.5b1" +__version__ = "0.8.5" From 68df355497cac714468eff0cdea30c270417f2bc Mon Sep 17 00:00:00 2001 From: Johannes Brachem Date: Tue, 20 Apr 2021 08:34:20 +0200 Subject: [PATCH 9/9] Update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbdd10e..ae0bcd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Mortimer v0.8.5 (Released 2021-04-20) + +### Changed v0.8.5 + +- Changed some internal functions in alfredo.py for compatibility with + alfred3 v2.0.0 + ## Mortimer v0.8.4 (Released 2021-04-15) ### Added v0.8.4