From 02b405ad69b2f3cb91a28966f0b586668598a461 Mon Sep 17 00:00:00 2001
From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com>
Date: Fri, 13 Dec 2024 11:47:51 -0500
Subject: [PATCH] FG-2933: Add recorder support for audio frames to MCAP
 recording demo

---
 website/package.json                          |  2 +-
 .../components/McapRecordingDemo/Recorder.ts  | 42 +++++++++++++++++++
 .../McapRecordingDemo/audioCapture.ts         |  6 +++
 yarn.lock                                     | 14 ++++++-
 4 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 website/src/components/McapRecordingDemo/audioCapture.ts

diff --git a/website/package.json b/website/package.json
index bdfb4fa619..e0d087be69 100644
--- a/website/package.json
+++ b/website/package.json
@@ -22,7 +22,7 @@
     "@docusaurus/preset-classic": "2.4.1",
     "@foxglove/eslint-plugin": "1.0.1",
     "@foxglove/rostime": "1.1.2",
-    "@foxglove/schemas": "1.6.2",
+    "@foxglove/schemas": "file:../../schemas",
     "@foxglove/tsconfig": "2.0.0",
     "@mcap/core": "workspace:*",
     "@mdx-js/react": "1.6.22",
diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts
index 058ae603c0..5a17d38ca9 100644
--- a/website/src/components/McapRecordingDemo/Recorder.ts
+++ b/website/src/components/McapRecordingDemo/Recorder.ts
@@ -3,6 +3,7 @@ import {
   PoseInFrame,
   CompressedImage,
   CompressedVideo,
+  CompressedAudio,
 } from "@foxglove/schemas";
 import { foxgloveMessageSchemas } from "@foxglove/schemas/internal";
 import zstd from "@foxglove/wasm-zstd";
@@ -11,6 +12,7 @@ import { EventEmitter } from "eventemitter3";
 import Queue from "promise-queue";
 
 import { ProtobufChannelInfo, addProtobufChannel } from "./addProtobufChannel";
+import { CompressedAudioFrame } from "./audioCapture";
 import { CompressedVideoFrame } from "./videoCapture";
 
 export type ProtobufObject<Message> = {
@@ -60,6 +62,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
   #vp9ChannelSeq = 0;
   #av1Channel?: ProtobufChannelInfo;
   #av1ChannelSeq = 0;
+  #opusChannel?: ProtobufChannelInfo;
+  #opusChannelSeq = 0;
 
   #blobParts: Uint8Array[] = [];
   bytesWritten = 0n;
@@ -112,6 +116,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     this.#h265ChannelSeq = 0;
     this.#av1Channel = undefined;
     this.#av1ChannelSeq = 0;
+    this.#opusChannel = undefined;
+    this.#opusChannelSeq = 0;
   }
 
   #time(): bigint {
@@ -277,6 +283,42 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     });
   }
 
+  async addAudioFrame(frame: CompressedAudioFrame): Promise<void> {
+    void this.#queue.add(async () => {
+      if (!this.#writer) {
+        return;
+      }
+      let channel: ProtobufChannelInfo;
+      let sequence: number;
+      switch (frame.format) {
+        case "opus":
+          channel = this.#opusChannel ??= await addProtobufChannel(
+            this.#writer,
+            "microphone_opus",
+            foxgloveMessageSchemas.CompressedAudio,
+          );
+          sequence = this.#opusChannelSeq++;
+          break;
+      }
+      const { id, rootType } = channel;
+      const now = this.#time();
+      const msg: ProtobufObject<CompressedAudio> = {
+        timestamp: toProtobufTime(fromNanoSec(now)),
+        data: new Uint8Array(await frame.blob.arrayBuffer()),
+        format: frame.format,
+      };
+      const data = rootType.encode(msg).finish();
+      await this.#writer.addMessage({
+        sequence,
+        channelId: id,
+        logTime: now,
+        publishTime: now,
+        data,
+      });
+      this.messageCount++;
+      this.#emit();
+    });
+  }
   async closeAndRestart(): Promise<Blob> {
     return await this.#queue.add(async () => {
       await this.#writer?.end();
diff --git a/website/src/components/McapRecordingDemo/audioCapture.ts b/website/src/components/McapRecordingDemo/audioCapture.ts
new file mode 100644
index 0000000000..8a442b7864
--- /dev/null
+++ b/website/src/components/McapRecordingDemo/audioCapture.ts
@@ -0,0 +1,6 @@
+type CompressedAudioFormat = "opus";
+export type CompressedAudioFrame = {
+  format: CompressedAudioFormat;
+  blob: Blob;
+};
+
diff --git a/yarn.lock b/yarn.lock
index 31ed8b2b47..f9158717e2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3056,6 +3056,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@foxglove/schemas@file:../../schemas::locator=website%40workspace%3Awebsite":
+  version: 1.6.7
+  resolution: "@foxglove/schemas@file:../../schemas#../../schemas::hash=4dd648&locator=website%40workspace%3Awebsite"
+  dependencies:
+    "@foxglove/rosmsg-msgs-common": "npm:^3.0.0"
+    tslib: "npm:^2.5.0"
+  checksum: 10c0/0f97472005ccbea8135a0a3effdbe10a95f8c21825f7b76d78cb3a2147986786af308008b07f6fda0776c5acc93eab5213062eb011c05937cfa57c448c8f3591
+  languageName: node
+  linkType: hard
+
 "@foxglove/schemas@npm:1.3.0":
   version: 1.3.0
   resolution: "@foxglove/schemas@npm:1.3.0"
@@ -3066,7 +3076,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@foxglove/schemas@npm:1.6.2, @foxglove/schemas@npm:^1.0.0":
+"@foxglove/schemas@npm:^1.0.0":
   version: 1.6.2
   resolution: "@foxglove/schemas@npm:1.6.2"
   dependencies:
@@ -15610,7 +15620,7 @@ __metadata:
     "@docusaurus/preset-classic": "npm:2.4.1"
     "@foxglove/eslint-plugin": "npm:1.0.1"
     "@foxglove/rostime": "npm:1.1.2"
-    "@foxglove/schemas": "npm:1.6.2"
+    "@foxglove/schemas": "file:../../schemas"
     "@foxglove/tsconfig": "npm:2.0.0"
     "@mcap/core": "workspace:*"
     "@mdx-js/react": "npm:1.6.22"