FG-2933: Add recorder support for audio frames to MCAP recording demo

foxglove · Dec 15, 2024 · db35b43 · db35b43
1 parent 4948ad4
commit db35b43
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 4 deletions.
diff --git a/website/package.json b/website/package.json
@@ -22,7 +22,7 @@
     "@docusaurus/preset-classic": "2.4.1",
     "@foxglove/eslint-plugin": "1.0.1",
     "@foxglove/rostime": "1.1.2",
-    "@foxglove/schemas": "1.6.2",
+    "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3",
     "@foxglove/tsconfig": "2.0.0",
     "@mcap/core": "workspace:*",
     "@mdx-js/react": "1.6.22",

diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts
@@ -1,8 +1,9 @@
-import { Time, fromNanoSec } from "@foxglove/rostime";
+import { Time, fromMillis, fromNanoSec } from "@foxglove/rostime";
 import {
   PoseInFrame,
   CompressedImage,
   CompressedVideo,
+  CompressedAudio,
 } from "@foxglove/schemas";
 import { foxgloveMessageSchemas } from "@foxglove/schemas/internal";
 import zstd from "@foxglove/wasm-zstd";
@@ -11,6 +12,7 @@ import { EventEmitter } from "eventemitter3";
 import Queue from "promise-queue";
 
 import { ProtobufChannelInfo, addProtobufChannel } from "./addProtobufChannel";
+import { CompressedAudioData } from "./audioCapture";
 import { CompressedVideoFrame } from "./videoCapture";
 
 export type ProtobufObject<Message> = {
@@ -60,6 +62,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
   #vp9ChannelSeq = 0;
   #av1Channel?: ProtobufChannelInfo;
   #av1ChannelSeq = 0;
+  #opusChannel?: ProtobufChannelInfo;
+  #opusChannelSeq = 0;
 
   #blobParts: Uint8Array[] = [];
   bytesWritten = 0n;
@@ -112,6 +116,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     this.#h265ChannelSeq = 0;
     this.#av1Channel = undefined;
     this.#av1ChannelSeq = 0;
+    this.#opusChannel = undefined;
+    this.#opusChannelSeq = 0;
   }
 
   #time(): bigint {
@@ -277,6 +283,45 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     });
   }
 
+  async addAudioData(data: CompressedAudioData): Promise<void> {
+    void this.#queue.add(async () => {
+      if (!this.#writer) {
+        return;
+      }
+      let channel: ProtobufChannelInfo;
+      let sequence: number;
+      switch (data.format) {
+        case "opus":
+          channel = this.#opusChannel ??= await addProtobufChannel(
+            this.#writer,
+            "microphone_opus",
+            foxgloveMessageSchemas.CompressedAudio,
+          );
+          sequence = this.#opusChannelSeq++;
+          break;
+      }
+      const { id, rootType } = channel;
+      const msg: ProtobufObject<CompressedAudio> = {
+        timestamp: toProtobufTime(fromMillis(data.timestamp)),
+        data: data.data,
+        format: data.format,
+        type: data.type,
+        sample_rate: data.sampleRate,
+        number_of_channels: data.numberOfChannels,
+      };
+      const now = this.#time();
+      await this.#writer.addMessage({
+        sequence,
+        channelId: id,
+        logTime: now,
+        publishTime: now,
+        data: rootType.encode(msg).finish(),
+      });
+      this.messageCount++;
+      this.#emit();
+    });
+  }
+
   async closeAndRestart(): Promise<Blob> {
     return await this.#queue.add(async () => {
       await this.#writer?.end();

diff --git a/website/src/components/McapRecordingDemo/audioCapture.ts b/website/src/components/McapRecordingDemo/audioCapture.ts
@@ -0,0 +1,13 @@
+type CompressedAudioFormat = "opus";
+type CompressedAudioType = "key" | "delta";
+export type CompressedAudioData = {
+  format: CompressedAudioFormat;
+  type: CompressedAudioType;
+  timestamp: number;
+  data: Uint8Array;
+  sampleRate: number;
+  numberOfChannels: number;
+
+  /** Call this function to release the buffer so it can be reused for new frames */
+  release: () => void;
+};
diff --git a/yarn.lock b/yarn.lock
@@ -3056,6 +3056,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@foxglove/schemas@github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3":
+  version: 1.6.7
+  resolution: "@foxglove/schemas@https://github.com/foxglove/schemas.git#commit=e2cd86e48762e9396737a00c56e1d45ebfae24e3"
+  dependencies:
+    "@foxglove/rosmsg-msgs-common": "npm:^3.0.0"
+    tslib: "npm:^2.5.0"
+  checksum: 10c0/67b13339aa3f485e786c93ef22f45f67598db1c77c0fa062b30c22a85f3be8a655ae5c351279bccd98751dce64ba7c95ddbd0cc7ca77b3281795e195faaa042d
+  languageName: node
+  linkType: hard
+
 "@foxglove/schemas@npm:1.3.0":
   version: 1.3.0
   resolution: "@foxglove/schemas@npm:1.3.0"
@@ -3066,7 +3076,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@foxglove/schemas@npm:1.6.2, @foxglove/schemas@npm:^1.0.0":
+"@foxglove/schemas@npm:^1.0.0":
   version: 1.6.2
   resolution: "@foxglove/schemas@npm:1.6.2"
   dependencies:
@@ -15610,7 +15620,7 @@ __metadata:
     "@docusaurus/preset-classic": "npm:2.4.1"
     "@foxglove/eslint-plugin": "npm:1.0.1"
     "@foxglove/rostime": "npm:1.1.2"
-    "@foxglove/schemas": "npm:1.6.2"
+    "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3"
     "@foxglove/tsconfig": "npm:2.0.0"
     "@mcap/core": "workspace:*"
     "@mdx-js/react": "npm:1.6.22"