Google Sheets export for form data

onadata/apps/main/tests/fixtures/google_sheets_export/expected_children.csv

@@ -0,0 +1,3 @@
+children/childs_name,children/childs_age,_id,_uuid,_submission_time,_index,_parent_table_name,_parent_index,_tags,_notes
+Tom,12,,,,1,tutorial_w_repeats,1,,
+Dick,5,,,,2,tutorial_w_repeats,1,,

onadata/apps/main/tests/fixtures/google_sheets_export/expected_choices.csv

@@ -0,0 +1,8 @@
+list name,name,label
+yes_no,0,no
+yes_no,1,yes
+,,
+browsers,firefox,Mozilla Firefox
+browsers,chrome,Google Chrome
+browsers,ie,Internet Explorer
+browsers,safari,Safari

onadata/apps/main/tests/fixtures/google_sheets_export/expected_flattened_raw.csv

@@ -0,0 +1,2 @@
+name,age,picture,has_children,children[1]/childs_age,children[1]/childs_name,children[2]/childs_age,children[2]/childs_name,gps,_gps_latitude,_gps_longitude,_gps_altitude,_gps_precision,web_browsers,meta/instanceID,_uuid,_submission_time,_tags,_notes
+Bob,25,,1,12,Tom,5,Dick,-1.2625621 36.7921711 0.0 20.0,-1.2625621,36.7921711,0.0,20.0,,uuid:b31c6ac2-b8ca-4180-914f-c844fa10ed3b,b31c6ac2-b8ca-4180-914f-c844fa10ed3b,2013-02-18T15:54:01,,

onadata/apps/main/tests/fixtures/google_sheets_export/expected_survey.csv

@@ -0,0 +1,11 @@
+type,name,label
+text,name,1. What is your name?
+integer,age,2. How old are you?
+image,picture,3. May I take your picture?
+select one from yes_no,has_children,4. Do you have any children?
+begin repeat,children,5. Children
+text,childs_name,5.1 Childs name?
+integer,childs_age,5.2 Childs age?
+end repeat,,
+geopoint,gps,5. Record your GPS coordinates.
+select all that apply from browsers,web_browsers,6. What web browsers do you use?

onadata/apps/main/tests/fixtures/google_sheets_export/expected_tutorial_w_repeats.csv

@@ -0,0 +1,2 @@
+name,age,picture,has_children,gps,_gps_latitude,_gps_longitude,_gps_altitude,_gps_precision,web_browsers,web_browsers/firefox,web_browsers/chrome,web_browsers/ie,web_browsers/safari,meta/instanceID,_id,_uuid,_submission_time,_index,_parent_table_name,_parent_index,_tags,_notes
+Bob,25,,1,-1.2625621 36.7921711 0.0 20.0,-1.2625621,36.7921711,0.0,20.0,,,,,,uuid:b31c6ac2-b8ca-4180-914f-c844fa10ed3b,###EXPORT_ID###,b31c6ac2-b8ca-4180-914f-c844fa10ed3b,2013-02-18T15:54:01,1,,-1,,

onadata/apps/main/tests/fixtures/google_sheets_export/tutorial_w_repeats.xml

@@ -0,0 +1 @@
+<?xml version='1.0' ?><tutorial_w_repeats id="tutorial_w_repeats"><name>Bob</name><age>25</age><picture /><has_children>1</has_children><children><childs_name>Tom</childs_name><childs_age>12</childs_age></children><children><childs_name>Dick</childs_name><childs_age>5</childs_age></children><gps>-1.2625621 36.7921711 0.0 20.0</gps><web_browsers /><meta><instanceID>uuid:b31c6ac2-b8ca-4180-914f-c844fa10ed3b</instanceID></meta></tutorial_w_repeats>

onadata/apps/main/tests/test_google_sheets_export.py

@@ -0,0 +1,157 @@
+import csv
+import os
+
+import gdata.gauth
+
+from django.conf import settings
+from django.core.files.storage import get_storage_class
+from django.core.files.temp import NamedTemporaryFile
+from django.core.urlresolvers import reverse
+from django.utils.dateparse import parse_datetime
+from mock import Mock, patch
+
+from onadata.apps.viewer.models.export import Export
+from onadata.libs.utils.export_tools import generate_export
+from onadata.libs.utils.google import oauth2_token
+from onadata.libs.utils.google_sheets import SheetsClient
+from test_base import TestBase
+
+
+
+class MockCell():
+    def __init__(self, row, col, value):
+        self.row = row
+        self.col = col
+        self.value = value
+
+
+class TestExport(TestBase):
+
+    def setUp(self):
+        # Prepare a fake token.
+        self.token = oauth2_token
+        self.token.refresh_token = 'foo'
+        self.token.access_token = 'bar'
+        self.token_blob = gdata.gauth.token_to_blob(self.token)
+
+        # Files that contain the expected spreadsheet data.
+        self.fixture_dir = os.path.join(
+            self.this_directory, 'fixtures', 'google_sheets_export')
+
+        # Create a test user and login.
+        self._create_user_and_login()
+
+        # Create a test submission.
+        path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
+        self._publish_xls_file_and_set_xform(path)
+        path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
+        self._submission_time = parse_datetime('2013-02-18 15:54:01Z')
+        self._make_submission(
+            path, forced_submission_time=self._submission_time)
+
+
+    def _mock_worksheet(self, csv_writer):
+        """Creates a mock worksheet object with append_row and insert_row 
+        methods writing to csv_writer."""
+        worksheet = Mock()
+        worksheet.append_row.side_effect = \
+            lambda values: csv_writer.writerow(values)
+        def create_cell(r, c):
+            return MockCell(r, c, None)
+        worksheet.cell.side_effect = create_cell
+        worksheet.update_cells.side_effect = \
+            lambda cells: csv_writer.writerow([cell.value for cell in cells])
+        worksheet.insert_row.side_effect = \
+            lambda values, index: csv_writer.writerow(values) 
+        return worksheet
+
+    def _mock_spreadsheet(self, csv_writers):
+        spreadsheet = Mock()
+        spreadsheet.add_worksheet.side_effect = \
+            [self._mock_worksheet(writer) for writer in csv_writers]
+        return spreadsheet
+
+    def _setup_result_files(self, expected_file_names):
+        expected_files = [open(os.path.join(self.fixture_dir, f)) 
+                          for f in expected_file_names]
+        result_files = [NamedTemporaryFile() for f in expected_file_names]
+        csv_writers = [csv.writer(f, lineterminator='\n') for f in result_files]
+        return expected_files, result_files, csv_writers
+
+    def assertStorageExists(self, export):
+        storage = get_storage_class()()
+        self.assertTrue(storage.exists(export.filepath))
+        _, ext = os.path.splitext(export.filename)
+        self.assertEqual(ext, '.gsheets')
+
+    def assertEqualExportFiles(self, expected_files, result_files, export):
+        for result, expected in zip(result_files, expected_files):
+            result.flush()
+            result.seek(0)
+            expected_content = expected.read()
+            # Fill in the actual export id (varies based on test order)
+            expected_content = expected_content.replace('###EXPORT_ID###', 
+                                                        str(export.id))
+            result_content = result.read()
+            self.assertEquals(result_content, expected_content)
+
+
+    @patch.object(SheetsClient, 'new')
+    @patch.object(SheetsClient, 'add_service_account_to_spreadsheet')
+    @patch.object(SheetsClient, 'get_worksheets_feed')
+    @patch('urllib2.urlopen')
+    def test_gsheets_export_output(self, 
+                                   mock_urlopen, 
+                                   mock_get_worksheets,
+                                   mock_account_add_service_account, 
+                                   mock_new):
+        expected_files, result_files, csv_writers = self._setup_result_files(
+            ['expected_tutorial_w_repeats.csv',
+             'expected_children.csv',
+             'expected_survey.csv',
+             'expected_choices.csv'])
+        mock_urlopen.return_value.read.return_value = '{"access_token": "baz"}'
+        mock_new.return_value = self._mock_spreadsheet(csv_writers)
+
+        # Test Google Sheets export.
+        export = generate_export(export_type=Export.GSHEETS_EXPORT, 
+                                 extension='gsheets', 
+                                 username=self.user.username, 
+                                 id_string='tutorial_w_repeats',
+                                 split_select_multiples=True,
+                                 binary_select_multiples=False,
+                                 google_token=self.token_blob,
+                                 flatten_repeated_fields=False,
+                                 export_xlsform=True)
+        self.assertStorageExists(export)
+        self.assertEqualExportFiles(expected_files, result_files, export)
+
+
+    @patch.object(SheetsClient, 'new')
+    @patch.object(SheetsClient, 'add_service_account_to_spreadsheet')
+    @patch.object(SheetsClient, 'get_worksheets_feed')
+    @patch('urllib2.urlopen')
+    def test_gsheets_export_flattened_output(self, 
+                                             mock_urlopen, 
+                                             mock_get_worksheets,
+                                             mock_account_add_service_account, 
+                                             mock_new):
+        expected_files, result_files, csv_writers = self._setup_result_files(
+            ['expected_flattened_raw.csv'])
+        mock_urlopen.return_value.read.return_value = '{"access_token": "baz"}'
+        mock_spreadsheet = self._mock_spreadsheet(csv_writers)
+        mock_new.return_value = mock_spreadsheet
+
+        # Test Google Sheets export.
+        export = generate_export(export_type=Export.GSHEETS_EXPORT, 
+                                 extension='gsheets', 
+                                 username=self.user.username, 
+                                 id_string='tutorial_w_repeats',
+                                 split_select_multiples=False,
+                                 binary_select_multiples=False,
+                                 google_token=self.token_blob,
+                                 flatten_repeated_fields=True,
+                                 export_xlsform=False)
+        self.assertStorageExists(export)
+        self.assertEqualExportFiles(expected_files, result_files, export)
+

onadata/apps/main/urls.py

@@ -152,8 +152,6 @@
         'onadata.apps.viewer.views.kml_export'),
     url(r"^(?P<username>\w+)/forms/(?P<id_string>[^/]+)/data\.zip",
         'onadata.apps.viewer.views.zip_export'),
-    url(r"^(?P<username>\w+)/forms/(?P<id_string>[^/]+)/gdocs$",
-        'onadata.apps.viewer.views.google_xls_export'),
     url(r"^(?P<username>\w+)/forms/(?P<id_string>[^/]+)/map_embed",
         'onadata.apps.viewer.views.map_embed_view'),
     url(r"^(?P<username>\w+)/forms/(?P<id_string>[^/]+)/map",

onadata/apps/viewer/models/export.py

@@ -28,7 +28,7 @@ def __str__(self):
     CSV_EXPORT = 'csv'
     KML_EXPORT = 'kml'
     ZIP_EXPORT = 'zip'
-    GDOC_EXPORT = 'gdoc'
+    GSHEETS_EXPORT = 'gsheets'
     CSV_ZIP_EXPORT = 'csv_zip'
     SAV_ZIP_EXPORT = 'sav_zip'
     SAV_EXPORT = 'sav'
@@ -48,7 +48,7 @@ def __str__(self):
     EXPORT_TYPES = [
         (XLS_EXPORT, 'Excel'),
         (CSV_EXPORT, 'CSV'),
-        (GDOC_EXPORT, 'GDOC'),
+        (GSHEETS_EXPORT, 'Google Sheets'),
         (ZIP_EXPORT, 'ZIP'),
         (KML_EXPORT, 'kml'),
         (CSV_ZIP_EXPORT, 'CSV ZIP'),

onadata/apps/viewer/tasks.py

@@ -30,9 +30,8 @@ def _create_export(xform, export_type):
         'export_id': export.id,
         'query': query,
     }
-    if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT,
-                       Export.CSV_EXPORT, Export.CSV_ZIP_EXPORT,
-                       Export.SAV_ZIP_EXPORT]:
+    if export_type in [Export.XLS_EXPORT, Export.CSV_EXPORT, 
+                       Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]:
         if options and "group_delimiter" in options:
             arguments["group_delimiter"] = options["group_delimiter"]
         if options and "split_select_multiples" in options:
@@ -43,7 +42,7 @@ def _create_export(xform, export_type):
                 options["binary_select_multiples"]
 
         # start async export
-        if export_type in [Export.XLS_EXPORT, Export.GDOC_EXPORT]:
+        if export_type == Export.XLS_EXPORT:
             result = create_xls_export.apply_async((), arguments, countdown=10)
         elif export_type == Export.CSV_EXPORT:
             result = create_csv_export.apply_async(
@@ -56,6 +55,23 @@ def _create_export(xform, export_type):
                 (), arguments, countdown=10)
         else:
             raise Export.ExportTypeError
+    elif export_type == Export.GSHEETS_EXPORT:
+        if options and "group_delimiter" in options:
+            arguments["group_delimiter"] = options["group_delimiter"]
+        if options and "split_select_multiples" in options:
+            arguments["split_select_multiples"] =\
+                options["split_select_multiples"]
+        if options and "binary_select_multiples" in options:
+            arguments["binary_select_multiples"] =\
+                options["binary_select_multiples"]
+        if options and "google_token" in options:
+            arguments["google_token"] = options["google_token"]
+        if options and "flatten_repeated_fields" in options:
+            arguments["flatten_repeated_fields"] =\
+                options["flatten_repeated_fields"]
+        if options and "export_xlsform" in options:
+            arguments["export_xlsform"] = options["export_xlsform"]
+        result = create_gsheets_export.apply_async((), arguments, countdown=10)
     elif export_type == Export.ZIP_EXPORT:
         # start async export
         result = create_zip_export.apply_async(
@@ -125,6 +141,44 @@ def create_xls_export(username, id_string, export_id, query=None,
     else:
         return gen_export.id
 
+@task()
+def create_gsheets_export(
+    username, id_string, export_id, query=None, group_delimiter='/', 
+    split_select_multiples=True, binary_select_multiples=False, 
+    google_token=None, flatten_repeated_fields=True, export_xlsform=True):
+    # we re-query the db instead of passing model objects according to
+    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
+    try:
+        export = Export.objects.get(id=export_id)
+    except Export.DoesNotExist:
+        # no export for this ID return None.
+        return None
+
+    # though export is not available when for has 0 submissions, we
+    # catch this since it potentially stops celery
+    try:
+        gen_export = generate_export(
+            Export.GSHEETS_EXPORT, None, username, id_string, export_id, query,
+            group_delimiter, split_select_multiples, binary_select_multiples,
+            google_token, flatten_repeated_fields, export_xlsform)
+    except (Exception, NoRecordsFoundError) as e:
+        export.internal_status = Export.FAILED
+        export.save()
+        # mail admins
+        details = {
+            'export_id': export_id,
+            'username': username,
+            'id_string': id_string
+        }
+        report_exception("Google Sheets Export Exception: Export ID - "
+                         "%(export_id)s, /%(username)s/%(id_string)s"
+                         % details, e, sys.exc_info())
+        # Raise for now to let celery know we failed
+        # - doesnt seem to break celery`
+        raise
+    else:
+        return gen_export.id
+
 
 @task()
 def create_csv_export(username, id_string, export_id, query=None,