Retry if unordered msg failure

ovotech · Aug 17, 2023 · eb77184 · eb77184
1 parent b52416f
commit eb77184
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 35 deletions.
diff --git a/docs/cli/unordered-messages.md b/docs/cli/unordered-messages.md
@@ -0,0 +1 @@
+# Handling Unordered Messages
diff --git a/packages/genesys-web-messaging-tester-cli/src/ScenarioResult.ts b/packages/genesys-web-messaging-tester-cli/src/ScenarioResult.ts
@@ -6,6 +6,7 @@ import {
 import { createConversationIdGetter } from './genesysPlatform/messageIdToConversationIdFactory';
 
 export interface ScenarioResult {
+  wasRetriedDueToUnorderedMessageFailure: boolean;
   scenario: TestScriptScenario;
   transcription: TranscribedMessage[];
   conversationId:

diff --git a/packages/genesys-web-messaging-tester-cli/src/cli.ts b/packages/genesys-web-messaging-tester-cli/src/cli.ts
@@ -8,6 +8,7 @@ import { createYamlFileReader } from './fileSystem/yamlFileReader';
 import { extractScenarios } from './testScript/parseTestScript';
 import {
   Conversation,
+  ReorderedMessageDelayer,
   SessionConfig,
   SessionTranscriber,
   TranscribedMessage,
@@ -24,6 +25,7 @@ import {
   messageIdToConversationIdFactory,
   MessageIdToConvoIdClient,
 } from './genesysPlatform/messageIdToConversationIdFactory';
+import { RetryTask, tryableTask } from './taskRetry';
 
 function parsePositiveInt(value: string) {
   const parsedValue = parseInt(value, 10);
@@ -45,7 +47,11 @@ export interface Dependencies {
   outputConsole?: Console;
   program?: Command;
   ui?: Ui;
-  webMessengerSessionFactory?: (sessionConfig: SessionConfig) => WebMessengerSession;
+  reorderedMessageDelayerFactory?: (delayBeforeEmittingInMs: number) => ReorderedMessageDelayer;
+  webMessengerSessionFactory?: (
+    sessionConfig: SessionConfig,
+    reorderedMessageDelayer: ReorderedMessageDelayer,
+  ) => WebMessengerSession;
   conversationFactory?: (session: WebMessengerSession) => Conversation;
   fsReadFileSync?: typeof readFileSync;
   fsAccessSync?: typeof accessSync;
@@ -69,7 +75,10 @@ export type Cli = (args: string[]) => Promise<void>;
 export function createCli({
   program = new Command(),
   ui = new Ui(),
-  webMessengerSessionFactory = (config) => new WebMessengerGuestSession(config, { IsTest: 'true' }),
+  reorderedMessageDelayerFactory = (delayBeforeEmittingInMs) =>
+    new ReorderedMessageDelayer(delayBeforeEmittingInMs),
+  webMessengerSessionFactory = (config, reorderedMessageDelayer) =>
+    new WebMessengerGuestSession(config, { IsTest: 'true' }, reorderedMessageDelayer),
   conversationFactory = (session) => new Conversation(session),
   fsReadFileSync = readFileSync,
   fsAccessSync = accessSync,
@@ -205,17 +214,22 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
     const tasks = new Listr<ListrRunContext>(
       testScriptScenarios.map((scenario) => ({
         title: ui?.titleOfTask(scenario),
-        task: async (context, task) => {
-          const transcription: TranscribedMessage[] = [];
-          const session = webMessengerSessionFactory(scenario.sessionConfig);
+        task: async (context, task) =>
+          tryableTask(async (isRetry) => {
+            const transcription: TranscribedMessage[] = [];
+
+            const reorderedMessageDelayer = reorderedMessageDelayerFactory(isRetry ? 7000 : 1000);
+            const session = webMessengerSessionFactory(
+              scenario.sessionConfig,
+              reorderedMessageDelayer,
+            );
 
-          let conversationIdGetter: ReturnType<typeof createConversationIdGetter> | undefined =
-            undefined;
-          if (associateId.enabled) {
-            conversationIdGetter = createConversationIdGetter(session, associateId.client);
-          }
+            let conversationIdGetter: ReturnType<typeof createConversationIdGetter> | undefined =
+              undefined;
+            if (associateId.enabled) {
+              conversationIdGetter = createConversationIdGetter(session, associateId.client);
+            }
 
-          try {
             new SessionTranscriber(session).on(
               'messageTranscribed',
               (event: TranscribedMessage) => {
@@ -239,35 +253,49 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
             const convo = conversationFactory(session);
             await convo.waitForConversationToStart();
 
-            for (const step of scenario.steps) {
-              await step(convo, { timeoutInSeconds: options.timeout });
+            let stepError: unknown;
+            try {
+              for (const step of scenario.steps) {
+                await step(convo, { timeoutInSeconds: options.timeout });
+              }
+            } catch (e) {
+              if (!isRetry && reorderedMessageDelayer.unorderdMessageDetected) {
+                task.output = ui?.retryingTestDueToFailureLikelyByUnorderedMessage();
+                task.title = ui?.titleOfTask(scenario, true);
+                throw new RetryTask();
+              } else {
+                stepError = e;
+              }
             }
-          } catch (error) {
+
+            if (stepError) {
+              session.close();
+
+              context.scenarioResults.push({
+                scenario,
+                transcription,
+                wasRetriedDueToUnorderedMessageFailure: isRetry,
+                hasPassed: false,
+                reasonForError:
+                  stepError instanceof Error ? stepError : new Error('Unexpected error occurred'),
+                conversationId: conversationIdGetter
+                  ? { associateId: true, conversationIdGetter }
+                  : { associateId: false },
+              });
+              throw new Error(ui?.titleOfFinishedTask(scenario, false));
+            }
+
+            task.title = ui?.titleOfFinishedTask(scenario, true);
             context.scenarioResults.push({
               scenario,
               transcription,
-              hasPassed: false,
-              reasonForError:
-                error instanceof Error ? error : new Error('Unexpected error occurred'),
+              wasRetriedDueToUnorderedMessageFailure: isRetry,
+              hasPassed: true,
               conversationId: conversationIdGetter
                 ? { associateId: true, conversationIdGetter }
                 : { associateId: false },
             });
-            throw new Error(ui?.titleOfFinishedTask(scenario, false));
-          } finally {
-            session.close();
-          }
-
-          task.title = ui?.titleOfFinishedTask(scenario, true);
-          context.scenarioResults.push({
-            scenario,
-            transcription,
-            hasPassed: true,
-            conversationId: conversationIdGetter
-              ? { associateId: true, conversationIdGetter }
-              : { associateId: false },
-          });
-        },
+          }),
       })),
       {
         concurrent: options.parallel,

diff --git a/packages/genesys-web-messaging-tester-cli/src/taskRetry.ts b/packages/genesys-web-messaging-tester-cli/src/taskRetry.ts
@@ -0,0 +1,16 @@
+export class RetryTask extends Error {
+  constructor() {
+    super('Retrying');
+    Object.setPrototypeOf(this, RetryTask.prototype);
+  }
+}
+
+export async function tryableTask(func: (isRetry: boolean) => Promise<void>): Promise<void> {
+  try {
+    await func(false);
+  } catch (e) {
+    if (e instanceof RetryTask) {
+      await func(true);
+    }
+  }
+}
diff --git a/packages/genesys-web-messaging-tester-cli/src/ui.ts b/packages/genesys-web-messaging-tester-cli/src/ui.ts
@@ -28,8 +28,15 @@ export class Ui {
     return Ui.trailingNewline(chalk.red(error.message));
   }
 
-  public titleOfTask(scenario: TestScriptScenario): string {
-    return scenario.name;
+  public titleOfTask(
+    scenario: TestScriptScenario,
+    isRetryDueToUnorderedMsgFailure = false,
+  ): string {
+    if (isRetryDueToUnorderedMsgFailure) {
+      return `${scenario.name} (RETRYING)`;
+    } else {
+      return scenario.name;
+    }
   }
 
   public titleOfFinishedTask(scenario: TestScriptScenario, hasPassed: boolean): string {
@@ -44,6 +51,10 @@ export class Ui {
     return Ui.trailingNewline(`${chalk.bold.grey(`${event.who}:`)} ${chalk.grey(event.message)}`);
   }
 
+  public retryingTestDueToFailureLikelyByUnorderedMessage(): string {
+    return Ui.trailingNewline('Test failed. Retrying as unordered messages detected');
+  }
+
   public firstLineOfMessageTranscribed(event: TranscribedMessage): string {
     const lines = event.message.trim().split('\n');
 
@@ -205,6 +216,15 @@ export class Ui {
       } else {
         lines.push(chalk.red(`FAIL - ${r.scenario.name}`));
       }
+
+      if (r.wasRetriedDueToUnorderedMessageFailure) {
+        lines.push(
+          chalk.yellow(
+            '  ^This test was retried following a failure that coincided with unordered messages being being received from Genesys\n' +
+              '  Read more here: https://github.com/ovotech/genesys-web-messaging-tester/blob/main/docs/cli/unordered-messages.md',
+          ),
+        );
+      }
     }
 
     return Ui.trailingNewline(

diff --git a/packages/genesys-web-messaging-tester/__tests__/fixtures/NoDelay.ts b/packages/genesys-web-messaging-tester/__tests__/fixtures/NoDelay.ts
@@ -1,6 +1,6 @@
 import { EventEmitter } from 'events';
 import { Response } from '../../src/genesys/Response';
-import { MessageDelayer } from '../../src/genesys/message-delayer/MessageDelayer';
+import { MessageDelayer } from '../../src';
 
 export class NoDelay extends EventEmitter implements MessageDelayer {
   constructor() {

diff --git a/packages/genesys-web-messaging-tester/src/genesys/message-delayer/ReorderedMessageDelayer.ts b/packages/genesys-web-messaging-tester/src/genesys/message-delayer/ReorderedMessageDelayer.ts
@@ -29,6 +29,8 @@ export class ReorderedMessageDelayer extends EventEmitter implements MessageDela
 
   private intervalReference: NodeJS.Timeout | undefined;
 
+  private unorderedMessageOccurred = false;
+
   constructor(
     private readonly delayBeforeEmittingInMs: number = 1000,
     private readonly intervalInMs: number = 1000,
@@ -49,6 +51,7 @@ export class ReorderedMessageDelayer extends EventEmitter implements MessageDela
           new Date(this.lastMessageWithTimestamp.body.channel.time).getTime();
 
         if (timeDifference < 0) {
+          this.unorderedMessageOccurred = true;
           ReorderedMessageDelayer.debugger(
             "Message received was out of order. Last msg's timestamp came before this one by %d ms",
             -timeDifference,
@@ -63,6 +66,10 @@ export class ReorderedMessageDelayer extends EventEmitter implements MessageDela
     }
   }
 
+  public get unorderdMessageDetected(): boolean {
+    return this.unorderedMessageOccurred;
+  }
+
   /**
    * Add a message to the pool. Each message added reset a timer to wait for any other messages
    * before releasing the oldest message.