Merge pull request #58 from jrjohnson/export-json

Store JSON survey data

handler.js

@@ -16,23 +16,31 @@ module.exports.storeSurveys = async event => {
 };
 
 const storeSurvey = async (surveyId) => {
+  await Promise.all([
+    storeSurveyType(surveyId, 'json'),
+    storeSurveyType(surveyId, 'csv'),
+  ]);
+  return `Stored ${surveyId}`;
+};
+
+const storeSurveyType = async (surveyId, type) => {
   console.log(`getting data for survey: ${surveyId}`);
-  const fileName = `/tmp/${surveyId}.csv`;
+  const fileName = `/tmp/${surveyId}.${type}`;
   const destinationStream = fs.createWriteStream(fileName);
   await getSurveyResults(
     process.env.QUALTRICS_API_TOKEN,
     process.env.QUALTRICS_DATA_CENTER,
     surveyId,
-    destinationStream
+    destinationStream,
+    type
   );
 
-  console.log(`survey data extracted, writing to S3 bucket ${process.env.BUCKET} ${surveyId}.csv`);
+  console.log(`survey data extracted, writing to S3 bucket ${process.env.BUCKET} ${surveyId}.${type}`);
   const params = {
     Bucket: process.env.BUCKET,
-    Key: `${surveyId}.csv`,
+    Key: `${surveyId}.${type}`,
     Body: fs.createReadStream(fileName)
   };
   await s3.upload(params).promise();
   console.log('done!');
-  return `Stored ${surveyId}`;
 };

index.js

@@ -9,13 +9,26 @@ const surveyId = process.argv[2];
 const dir = path.join(__dirname, 'output');
 fs.mkdirSync(dir, { recursive: true });
 
-const filePath = path.join(dir, `${surveyId}.csv`);
-const destinationStream = fs.createWriteStream(filePath);
-getSurveyResults(
+const jsonFilePath = path.join(dir, `${surveyId}.json`);
+const jsonDestinationStream = fs.createWriteStream(jsonFilePath);
+const jsonPromise = getSurveyResults(
   process.env.QUALTRICS_API_TOKEN,
   process.env.QUALTRICS_DATA_CENTER,
   surveyId,
-  destinationStream
-).then(() => {
+  jsonDestinationStream,
+  'json'
+);
+
+const csvFilePath = path.join(dir, `${surveyId}.csv`);
+const csvDestinationStream = fs.createWriteStream(csvFilePath);
+const csvPromise = getSurveyResults(
+  process.env.QUALTRICS_API_TOKEN,
+  process.env.QUALTRICS_DATA_CENTER,
+  surveyId,
+  csvDestinationStream,
+  'csv'
+);
+
+Promise.all([jsonPromise, csvPromise]).then(() => {
   console.log("done\n");
 });

src/get-survey-results.js

@@ -3,9 +3,9 @@ const fetch = require('node-fetch');
 const yauzl = require('yauzl-promise');
 const { sleep } = require('./sleep');
 
-async function requestResultsToBeBuilt(token, dataCenter, surveyId) {
+async function requestResultsToBeBuilt(token, dataCenter, surveyId, format) {
   const payload = {
-    format: 'csv',
+    format,
     surveyId
   };
   const url = `https://${dataCenter}.qualtrics.com/API/v3/responseexports`;
@@ -57,8 +57,8 @@ async function writeFile(token, url, destinationStream) {
   });
 }
 
-async function getSurveyResults(token, dataCenter, surveyId, destinationStream) {
-  const progressId = await requestResultsToBeBuilt(token, dataCenter, surveyId);
+async function getSurveyResults(token, dataCenter, surveyId, destinationStream, format) {
+  const progressId = await requestResultsToBeBuilt(token, dataCenter, surveyId, format);
   const fileUrl = await waitForBuildToComplete(token, dataCenter, progressId);
   return await writeFile(token, fileUrl, destinationStream);
 }