From db35b43966147c8cf9a5f9b65bc2789288aea7eb Mon Sep 17 00:00:00 2001 From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com> Date: Sun, 15 Dec 2024 14:34:57 -0500 Subject: [PATCH 1/4] FG-2933: Add recorder support for audio frames to MCAP recording demo --- website/package.json | 2 +- .../components/McapRecordingDemo/Recorder.ts | 47 ++++++++++++++++++- .../McapRecordingDemo/audioCapture.ts | 13 +++++ yarn.lock | 14 +++++- 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 website/src/components/McapRecordingDemo/audioCapture.ts diff --git a/website/package.json b/website/package.json index bdfb4fa619..750a5925c1 100644 --- a/website/package.json +++ b/website/package.json @@ -22,7 +22,7 @@ "@docusaurus/preset-classic": "2.4.1", "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rostime": "1.1.2", - "@foxglove/schemas": "1.6.2", + "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3", "@foxglove/tsconfig": "2.0.0", "@mcap/core": "workspace:*", "@mdx-js/react": "1.6.22", diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts index 058ae603c0..27f78021d4 100644 --- a/website/src/components/McapRecordingDemo/Recorder.ts +++ b/website/src/components/McapRecordingDemo/Recorder.ts @@ -1,8 +1,9 @@ -import { Time, fromNanoSec } from "@foxglove/rostime"; +import { Time, fromMillis, fromNanoSec } from "@foxglove/rostime"; import { PoseInFrame, CompressedImage, CompressedVideo, + CompressedAudio, } from "@foxglove/schemas"; import { foxgloveMessageSchemas } from "@foxglove/schemas/internal"; import zstd from "@foxglove/wasm-zstd"; @@ -11,6 +12,7 @@ import { EventEmitter } from "eventemitter3"; import Queue from "promise-queue"; import { ProtobufChannelInfo, addProtobufChannel } from "./addProtobufChannel"; +import { CompressedAudioData } from "./audioCapture"; import { CompressedVideoFrame } from "./videoCapture"; export type ProtobufObject = { @@ -60,6 +62,8 @@ export class Recorder extends EventEmitter { #vp9ChannelSeq = 0; #av1Channel?: ProtobufChannelInfo; #av1ChannelSeq = 0; + #opusChannel?: ProtobufChannelInfo; + #opusChannelSeq = 0; #blobParts: Uint8Array[] = []; bytesWritten = 0n; @@ -112,6 +116,8 @@ export class Recorder extends EventEmitter { this.#h265ChannelSeq = 0; this.#av1Channel = undefined; this.#av1ChannelSeq = 0; + this.#opusChannel = undefined; + this.#opusChannelSeq = 0; } #time(): bigint { @@ -277,6 +283,45 @@ export class Recorder extends EventEmitter { }); } + async addAudioData(data: CompressedAudioData): Promise { + void this.#queue.add(async () => { + if (!this.#writer) { + return; + } + let channel: ProtobufChannelInfo; + let sequence: number; + switch (data.format) { + case "opus": + channel = this.#opusChannel ??= await addProtobufChannel( + this.#writer, + "microphone_opus", + foxgloveMessageSchemas.CompressedAudio, + ); + sequence = this.#opusChannelSeq++; + break; + } + const { id, rootType } = channel; + const msg: ProtobufObject = { + timestamp: toProtobufTime(fromMillis(data.timestamp)), + data: data.data, + format: data.format, + type: data.type, + sample_rate: data.sampleRate, + number_of_channels: data.numberOfChannels, + }; + const now = this.#time(); + await this.#writer.addMessage({ + sequence, + channelId: id, + logTime: now, + publishTime: now, + data: rootType.encode(msg).finish(), + }); + this.messageCount++; + this.#emit(); + }); + } + async closeAndRestart(): Promise { return await this.#queue.add(async () => { await this.#writer?.end(); diff --git a/website/src/components/McapRecordingDemo/audioCapture.ts b/website/src/components/McapRecordingDemo/audioCapture.ts new file mode 100644 index 0000000000..e53ca5164c --- /dev/null +++ b/website/src/components/McapRecordingDemo/audioCapture.ts @@ -0,0 +1,13 @@ +type CompressedAudioFormat = "opus"; +type CompressedAudioType = "key" | "delta"; +export type CompressedAudioData = { + format: CompressedAudioFormat; + type: CompressedAudioType; + timestamp: number; + data: Uint8Array; + sampleRate: number; + numberOfChannels: number; + + /** Call this function to release the buffer so it can be reused for new frames */ + release: () => void; +}; diff --git a/yarn.lock b/yarn.lock index 31ed8b2b47..c252e3da11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3056,6 +3056,16 @@ __metadata: languageName: node linkType: hard +"@foxglove/schemas@github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3": + version: 1.6.7 + resolution: "@foxglove/schemas@https://github.com/foxglove/schemas.git#commit=e2cd86e48762e9396737a00c56e1d45ebfae24e3" + dependencies: + "@foxglove/rosmsg-msgs-common": "npm:^3.0.0" + tslib: "npm:^2.5.0" + checksum: 10c0/67b13339aa3f485e786c93ef22f45f67598db1c77c0fa062b30c22a85f3be8a655ae5c351279bccd98751dce64ba7c95ddbd0cc7ca77b3281795e195faaa042d + languageName: node + linkType: hard + "@foxglove/schemas@npm:1.3.0": version: 1.3.0 resolution: "@foxglove/schemas@npm:1.3.0" @@ -3066,7 +3076,7 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@npm:1.6.2, @foxglove/schemas@npm:^1.0.0": +"@foxglove/schemas@npm:^1.0.0": version: 1.6.2 resolution: "@foxglove/schemas@npm:1.6.2" dependencies: @@ -15610,7 +15620,7 @@ __metadata: "@docusaurus/preset-classic": "npm:2.4.1" "@foxglove/eslint-plugin": "npm:1.0.1" "@foxglove/rostime": "npm:1.1.2" - "@foxglove/schemas": "npm:1.6.2" + "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3" "@foxglove/tsconfig": "npm:2.0.0" "@mcap/core": "workspace:*" "@mdx-js/react": "npm:1.6.22" From 4eb2e80094e19e3cf16b20a0e457073969384cd6 Mon Sep 17 00:00:00 2001 From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com> Date: Sun, 15 Dec 2024 14:44:05 -0500 Subject: [PATCH 2/4] Release the buffer after encoding the message --- website/src/components/McapRecordingDemo/Recorder.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts index 27f78021d4..fa751e0f67 100644 --- a/website/src/components/McapRecordingDemo/Recorder.ts +++ b/website/src/components/McapRecordingDemo/Recorder.ts @@ -309,13 +309,15 @@ export class Recorder extends EventEmitter { sample_rate: data.sampleRate, number_of_channels: data.numberOfChannels, }; + const encodedMsg = rootType.encode(msg).finish(); + data.release(); const now = this.#time(); await this.#writer.addMessage({ sequence, channelId: id, logTime: now, publishTime: now, - data: rootType.encode(msg).finish(), + data: encodedMsg, }); this.messageCount++; this.#emit(); From 32a35acbc9cfacbc19d20ef0948e4aa17dfafa70 Mon Sep 17 00:00:00 2001 From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com> Date: Sun, 15 Dec 2024 14:49:03 -0500 Subject: [PATCH 3/4] cspell --- website/src/components/McapRecordingDemo/Recorder.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts index fa751e0f67..c72217239d 100644 --- a/website/src/components/McapRecordingDemo/Recorder.ts +++ b/website/src/components/McapRecordingDemo/Recorder.ts @@ -1,3 +1,5 @@ +// cspell:word millis + import { Time, fromMillis, fromNanoSec } from "@foxglove/rostime"; import { PoseInFrame, From 742d8f3036de34b078feb87fea14da53496fc493 Mon Sep 17 00:00:00 2001 From: fgwt202412 <191263616+fgwt202412@users.noreply.github.com> Date: Sun, 15 Dec 2024 18:01:30 -0500 Subject: [PATCH 4/4] Update @foxglove/schemas --- website/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/website/package.json b/website/package.json index 750a5925c1..4d3ff5397b 100644 --- a/website/package.json +++ b/website/package.json @@ -22,7 +22,7 @@ "@docusaurus/preset-classic": "2.4.1", "@foxglove/eslint-plugin": "1.0.1", "@foxglove/rostime": "1.1.2", - "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3", + "@foxglove/schemas": "github:foxglove/schemas#fbb9ccfd8d31ac6c3aa18ce7b9959eb0be9bed7c", "@foxglove/tsconfig": "2.0.0", "@mcap/core": "workspace:*", "@mdx-js/react": "1.6.22", diff --git a/yarn.lock b/yarn.lock index c252e3da11..51e27d0c96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3056,13 +3056,13 @@ __metadata: languageName: node linkType: hard -"@foxglove/schemas@github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3": +"@foxglove/schemas@github:foxglove/schemas#fbb9ccfd8d31ac6c3aa18ce7b9959eb0be9bed7c": version: 1.6.7 - resolution: "@foxglove/schemas@https://github.com/foxglove/schemas.git#commit=e2cd86e48762e9396737a00c56e1d45ebfae24e3" + resolution: "@foxglove/schemas@https://github.com/foxglove/schemas.git#commit=fbb9ccfd8d31ac6c3aa18ce7b9959eb0be9bed7c" dependencies: "@foxglove/rosmsg-msgs-common": "npm:^3.0.0" tslib: "npm:^2.5.0" - checksum: 10c0/67b13339aa3f485e786c93ef22f45f67598db1c77c0fa062b30c22a85f3be8a655ae5c351279bccd98751dce64ba7c95ddbd0cc7ca77b3281795e195faaa042d + checksum: 10c0/5153e9973b5d16ae92bdd6d4c1837518adc425f78a33985cf3f364027895c792d136aa6ac9f7233d44670b4be72bd4c3946aec3c7099514054af88eaf8ffde1b languageName: node linkType: hard @@ -15620,7 +15620,7 @@ __metadata: "@docusaurus/preset-classic": "npm:2.4.1" "@foxglove/eslint-plugin": "npm:1.0.1" "@foxglove/rostime": "npm:1.1.2" - "@foxglove/schemas": "github:foxglove/schemas#e2cd86e48762e9396737a00c56e1d45ebfae24e3" + "@foxglove/schemas": "github:foxglove/schemas#fbb9ccfd8d31ac6c3aa18ce7b9959eb0be9bed7c" "@foxglove/tsconfig": "npm:2.0.0" "@mcap/core": "workspace:*" "@mdx-js/react": "npm:1.6.22"