fix: Add tests for retry cancelled collection jobs (#1214)

* some fixes for retry cancelled

* add test for retry cancelled jobs

* fix for fetching test ids

* sort test ids array to prevent diff problems

Author: gnarf

client/components/ManageBotRunDialog/queries.js

@@ -52,6 +52,9 @@ export const RETRY_CANCELED_COLLECTIONS = gql`
       retryCanceledCollections {
         id
         status
+        testStatus {
+          status
+        }
       }
     }
   }

server/models/services/CollectionJobService.js

@@ -388,12 +388,13 @@ const getCollectionJobs = async ({
 const triggerWorkflow = async (job, testIds, atVersion, { transaction }) => {
   const { testPlanVersion } = job.testPlanRun.testPlanReport;
   const { gitSha, directory } = testPlanVersion;
+
   try {
     if (isGithubWorkflowEnabled()) {
       // TODO: pass the reduced list of testIds along / deal with them somehow
       await createGithubWorkflow({ job, directory, gitSha, atVersion });
     } else {
-      await startCollectionJobSimulation(job, atVersion, transaction);
+      await startCollectionJobSimulation(job, testIds, atVersion, transaction);
     }
   } catch (error) {
     console.error(error);
@@ -479,14 +480,12 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => {
     throw new Error('collectionJob is required to retry cancelled tests');
   }
 
-  const cancelledTests = collectionJob.testPlanRun.testResults.filter(
-    testResult =>
-      // Find tests that don't have complete output
-      !testResult?.scenarioResults?.every(scenario => scenario?.output !== null)
+  const cancelledTests = collectionJob.testStatus.filter(
+    testStatus => testStatus.status === COLLECTION_JOB_STATUS.CANCELLED
   );
 
   const testPlanReport = await getTestPlanReportById({
-    id: job.testPlanRun.testPlanReportId,
+    id: collectionJob.testPlanRun.testPlanReportId,
     transaction
   });
 
@@ -497,10 +496,22 @@ const retryCanceledCollections = async ({ collectionJob }, { transaction }) => {
     transaction
   );
 
-  const testIds = cancelledTests.map(test => test.id);
+  const testIds = cancelledTests.map(test => test.testId);
 
-  const job = await getCollectionJobById({
+  const job = await updateCollectionJobById({
     id: collectionJob.id,
+    values: { status: COLLECTION_JOB_STATUS.QUEUED },
+    transaction
+  });
+
+  await updateCollectionJobTestStatusByQuery({
+    where: {
+      collectionJobId: job.id,
+      status: COLLECTION_JOB_STATUS.CANCELLED
+    },
+    values: {
+      status: COLLECTION_JOB_STATUS.QUEUED
+    },
     transaction
   });
 
@@ -652,6 +663,15 @@ const restartCollectionJob = async ({ id }, { transaction }) => {
     },
     transaction
   });
+  await updateCollectionJobTestStatusByQuery({
+    where: {
+      collectionJobId: id
+    },
+    values: {
+      status: COLLECTION_JOB_STATUS.QUEUED
+    },
+    transaction
+  });
 
   if (!job) {
     return null;
@@ -669,7 +689,16 @@ const restartCollectionJob = async ({ id }, { transaction }) => {
     transaction
   );
 
-  return triggerWorkflow(job, [], atVersion, { transaction });
+  const tests = await runnableTestsResolver(testPlanReport, null, {
+    transaction
+  });
+
+  return triggerWorkflow(
+    job,
+    tests.map(test => test.id),
+    atVersion,
+    { transaction }
+  );
 };
 
 /**

server/tests/integration/automation-scheduler.test.js

@@ -1,7 +1,8 @@
 const startSupertestServer = require('../util/api-server');
 const automationRoutes = require('../../routes/automation');
 const {
-  setupMockAutomationSchedulerServer
+  setupMockAutomationSchedulerServer,
+  startCollectionJobSimulation
 } = require('../util/mock-automation-scheduler-server');
 const db = require('../../models/index');
 const { query, mutate } = require('../util/graphql-test-utilities');
@@ -197,6 +198,22 @@ const restartCollectionJobByMutation = async (jobId, { transaction }) =>
     { transaction }
   );
 
+const retryCanceledCollectionJobByMutation = async (jobId, { transaction }) =>
+  await mutate(
+    `
+  mutation {
+    collectionJob(id: "${jobId}") {
+      retryCanceledCollections {
+        id
+        status
+        testStatus {
+          status
+        }
+      }
+    }
+  }  `,
+    { transaction }
+  );
 const cancelCollectionJobByMutation = async (jobId, { transaction }) =>
   await mutate(
     `
@@ -237,6 +254,18 @@ describe('Automation controller', () => {
       expect(storedJob.status).toEqual('QUEUED');
       expect(storedJob.testPlanRun.testPlanReport.id).toEqual(testPlanReportId);
       expect(storedJob.testPlanRun.testResults.length).toEqual(0);
+      const collectionJob = await getCollectionJobById({
+        id: storedJob.id,
+        transaction
+      });
+      const tests =
+        collectionJob.testPlanRun.testPlanReport.testPlanVersion.tests.filter(
+          test => test.at.key === 'voiceover_macos'
+        );
+      // check testIds - order doesn't matter, so we sort them
+      expect(
+        startCollectionJobSimulation.lastCallParams.testIds.sort()
+      ).toEqual(tests.map(test => test.id).sort());
     });
   });
 
@@ -260,6 +289,105 @@ describe('Automation controller', () => {
     });
   });
 
+  it('should retry a cancelled job with only remaining tests', async () => {
+    await apiServer.sessionAgentDbCleaner(async transaction => {
+      const { scheduleCollectionJob: job } =
+        await scheduleCollectionJobByMutation({ transaction });
+      const collectionJob = await getCollectionJobById({
+        id: job.id,
+        transaction
+      });
+      const secret = await getJobSecret(collectionJob.id, { transaction });
+
+      // start "RUNNING" the job
+      const response1 = await sessionAgent
+        .post(`/api/jobs/${collectionJob.id}`)
+        .send({ status: 'RUNNING' })
+        .set('x-automation-secret', secret)
+        .set('x-transaction-id', transaction.id);
+      expect(response1.statusCode).toBe(200);
+
+      // simulate a response for a test
+      const automatedTestResponse = 'AUTOMATED TEST RESPONSE';
+      const ats = await AtLoader().getAll({ transaction });
+      const browsers = await BrowserLoader().getAll({ transaction });
+      const at = ats.find(
+        at => at.id === collectionJob.testPlanRun.testPlanReport.at.id
+      );
+      const browser = browsers.find(
+        browser =>
+          browser.id === collectionJob.testPlanRun.testPlanReport.browser.id
+      );
+      const { tests } =
+        collectionJob.testPlanRun.testPlanReport.testPlanVersion;
+      const selectedTestIndex = 2;
+
+      const selectedTest = tests[selectedTestIndex];
+      const selectedTestRowNumber = selectedTest.rowNumber;
+
+      const numberOfScenarios = selectedTest.scenarios.filter(
+        scenario => scenario.atId === at.id
+      ).length;
+      const response2 = await sessionAgent
+        .post(`/api/jobs/${collectionJob.id}/test/${selectedTestRowNumber}`)
+        .send({
+          capabilities: {
+            atName: at.name,
+            atVersion: at.atVersions[0].name,
+            browserName: browser.name,
+            browserVersion: browser.browserVersions[0].name
+          },
+          responses: new Array(numberOfScenarios).fill(automatedTestResponse)
+        })
+        .set('x-automation-secret', secret)
+        .set('x-transaction-id', transaction.id);
+      expect(response2.statusCode).toBe(200);
+      // cancel the job
+      const {
+        collectionJob: { cancelCollectionJob: cancelledCollectionJob }
+      } = await cancelCollectionJobByMutation(collectionJob.id, {
+        transaction
+      });
+
+      // check canceled status
+
+      expect(cancelledCollectionJob.status).toEqual('CANCELLED');
+      const { collectionJob: storedCollectionJob } = await getTestCollectionJob(
+        collectionJob.id,
+        { transaction }
+      );
+      expect(storedCollectionJob.status).toEqual('CANCELLED');
+      for (const test of storedCollectionJob.testStatus) {
+        const expectedStatus =
+          test.test.id == selectedTest.id ? 'COMPLETED' : 'CANCELLED';
+        expect(test.status).toEqual(expectedStatus);
+      }
+
+      // retry job
+      const data = await retryCanceledCollectionJobByMutation(
+        collectionJob.id,
+        { transaction }
+      );
+      expect(data.collectionJob.retryCanceledCollections.status).toBe('QUEUED');
+      const { collectionJob: restartedCollectionJob } =
+        await getTestCollectionJob(collectionJob.id, { transaction });
+      // check restarted status
+      expect(restartedCollectionJob.status).toEqual('QUEUED');
+      for (const test of restartedCollectionJob.testStatus) {
+        const expectedStatus =
+          test.test.id == selectedTest.id ? 'COMPLETED' : 'QUEUED';
+        expect(test.status).toEqual(expectedStatus);
+      }
+      const expectedTests = tests.filter(
+        test => test.at.key === 'voiceover_macos' && test.id != selectedTest.id
+      );
+      // check testIds - order doesn't matter, so we sort them
+      expect(
+        startCollectionJobSimulation.lastCallParams.testIds.sort()
+      ).toEqual(expectedTests.map(test => test.id).sort());
+    });
+  });
+
   it('should gracefully reject request to cancel a job that does not exist', async () => {
     await dbCleaner(async transaction => {
       expect.assertions(1); // Make sure an assertion is made

server/tests/util/mock-automation-scheduler-server.js

@@ -136,10 +136,18 @@ const simulateResultCompletion = async (
   }
 };
 
-const startCollectionJobSimulation = async (job, atVersion, transaction) => {
+const startCollectionJobSimulation = async (
+  job,
+  testIds,
+  atVersion,
+  transaction
+) => {
   if (!mockSchedulerEnabled) throw new Error('mock scheduler is not enabled');
   if (process.env.ENVIRONMENT === 'test') {
     // stub behavior in test suite
+
+    startCollectionJobSimulation.lastCallParams = { job, testIds, atVersion };
+
     return { status: COLLECTION_JOB_STATUS.QUEUED };
   } else {
     const { data } = await apolloServer.executeOperation(