Skip to content

Commit

Permalink
Add VP9 encoding to web demo if supported (#1288)
Browse files Browse the repository at this point in the history
Same as #1275 & #1278, this time for VP9. The number of checkboxes is
getting out of hand:

<img width="593" alt="image"
src="https://github.com/user-attachments/assets/2d366eff-694a-4c75-a80f-36c2a1403414">
  • Loading branch information
jtbandes authored Dec 11, 2024
1 parent a3b6880 commit 212b873
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 6 deletions.
20 changes: 19 additions & 1 deletion website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
supportsAV1Encoding,
supportsH264Encoding,
supportsH265Encoding,
supportsVP9Encoding,
} from "./videoCapture";

type State = {
Expand Down Expand Up @@ -130,6 +131,7 @@ export function McapRecordingDemo(): JSX.Element {
const [recordJpeg, setRecordJpeg] = useState(false);
const [recordH264, setRecordH264] = useState(false);
const [recordH265, setRecordH265] = useState(false);
const [recordVP9, setRecordVP9] = useState(false);
const [recordAV1, setRecordAV1] = useState(false);
const [recordMouse, setRecordMouse] = useState(true);
const [recordOrientation, setRecordOrientation] = useState(true);
Expand All @@ -142,12 +144,14 @@ export function McapRecordingDemo(): JSX.Element {

const { data: h264Support } = useAsync(supportsH264Encoding);
const { data: h265Support } = useAsync(supportsH265Encoding);
const { data: vp9Support } = useAsync(supportsVP9Encoding);
const { data: av1Support } = useAsync(supportsAV1Encoding);

const canStartRecording =
recordMouse ||
(!hasMouse && recordOrientation) ||
(recordAV1 && !videoError) ||
(recordVP9 && !videoError) ||
(recordH265 && !videoError) ||
(recordH264 && !videoError) ||
(recordJpeg && !videoError);
Expand Down Expand Up @@ -196,7 +200,8 @@ export function McapRecordingDemo(): JSX.Element {
};
}, [addPoseMessage, recording, recordOrientation]);

const enableCamera = recordAV1 || recordH265 || recordH264 || recordJpeg;
const enableCamera =
recordAV1 || recordVP9 || recordH265 || recordH264 || recordJpeg;
useEffect(() => {
const videoContainer = videoContainerRef.current;
if (!videoContainer || !enableCamera) {
Expand Down Expand Up @@ -243,6 +248,7 @@ export function McapRecordingDemo(): JSX.Element {
const stopCapture = startVideoCapture({
video,
enableAV1: recordAV1,
enableVP9: recordVP9,
enableH265: recordH265,
enableH264: recordH264,
enableJpeg: recordJpeg,
Expand Down Expand Up @@ -365,6 +371,18 @@ export function McapRecordingDemo(): JSX.Element {
Camera (AV1)
</label>
)}
{vp9Support?.supported === true && (
<label>
<input
type="checkbox"
checked={recordVP9}
onChange={(event) => {
setRecordVP9(event.target.checked);
}}
/>
Camera (VP9)
</label>
)}
{h265Support?.supported === true && (
<label>
<input
Expand Down
10 changes: 10 additions & 0 deletions website/src/components/McapRecordingDemo/Recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ export class Recorder extends EventEmitter<RecorderEvents> {
#h264ChannelSeq = 0;
#h265Channel?: ProtobufChannelInfo;
#h265ChannelSeq = 0;
#vp9Channel?: ProtobufChannelInfo;
#vp9ChannelSeq = 0;
#av1Channel?: ProtobufChannelInfo;
#av1ChannelSeq = 0;

Expand Down Expand Up @@ -236,6 +238,14 @@ export class Recorder extends EventEmitter<RecorderEvents> {
);
sequence = this.#h265ChannelSeq++;
break;
case "vp9":
channel = this.#vp9Channel ??= await addProtobufChannel(
this.#writer,
"camera_vp9",
foxgloveMessageSchemas.CompressedVideo,
);
sequence = this.#vp9ChannelSeq++;
break;
case "av1":
channel = this.#av1Channel ??= await addProtobufChannel(
this.#writer,
Expand Down
57 changes: 52 additions & 5 deletions website/src/components/McapRecordingDemo/videoCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ export function startVideoStream(params: VideoStreamParams): () => void {
};
}

type CompressedVideoFormat = "h264" | "h265" | "av1";
type CompressedVideoFormat = "h264" | "h265" | "vp9" | "av1";
export type CompressedVideoFrame = {
format: CompressedVideoFormat;
data: Uint8Array;
Expand All @@ -82,6 +82,7 @@ export type CompressedVideoFrame = {
type VideoCaptureParams = {
enableH265: boolean;
enableH264: boolean;
enableVP9: boolean;
enableAV1: boolean;
enableJpeg: boolean;
/** Video element to capture */
Expand Down Expand Up @@ -124,6 +125,11 @@ const BASE_CONFIG_H265: Omit<VideoEncoderConfig, "width" | "height"> = {
avc: { format: "annexb" }, // https://bugs.webkit.org/show_bug.cgi?id=281945
hevc: { format: "annexb" },
};
const BASE_CONFIG_VP9: Omit<VideoEncoderConfig, "width" | "height"> = {
// https://github.com/webmproject/vp9-dash/blob/main/VPCodecISOMediaFileFormatBinding.md
// Profile 0, level 3.1, 8 bits
codec: "vp09.00.31.08",
};
const BASE_CONFIG_AV1: Omit<VideoEncoderConfig, "width" | "height"> = {
// https://aomediacodec.github.io/av1-isobmff/
// https://jakearchibald.com/2022/html-codecs-parameter-for-av1/
Expand Down Expand Up @@ -181,6 +187,31 @@ export async function supportsH265Encoding(): Promise<{
};
}

/**
* Determine whether VideoEncoder can (probably) be used to encode video with VP9.
*/
export async function supportsVP9Encoding(): Promise<{
supported: boolean;
/** True if too many keyframes may be produced (e.g. https://bugs.webkit.org/show_bug.cgi?id=264893) */
mayUseLotsOfKeyframes: boolean;
}> {
const result = await selectSupportedVideoEncoderConfig({
baseConfig: {
...BASE_CONFIG_VP9,
// Notes about fake width/height:
// - Some platforms require them to be even numbers
// - Too small or too large return false from isConfigSupported in Chrome
width: 640,
height: 480,
},
frameDurationSec: 1 / 30,
});
return {
supported: result != undefined,
mayUseLotsOfKeyframes: result?.mayUseLotsOfKeyframes ?? false,
};
}

/**
* Determine whether VideoEncoder can (probably) be used to encode video with AV1.
*/
Expand Down Expand Up @@ -425,18 +456,19 @@ async function startVideoCaptureAsync(
) {
const {
video,
enableH265,
enableH264,
enableJpeg,
enableH265,
enableVP9,
enableAV1,
enableJpeg,
onJpegFrame,
onVideoFrame,
onError,
frameDurationSec,
} = params;
if (!enableAV1 && !enableH265 && !enableH264 && !enableJpeg) {
if (!enableAV1 && !enableVP9 && !enableH265 && !enableH264 && !enableJpeg) {
throw new Error(
"At least one of AV1, H.265, H.264, or JPEG encoding must be enabled",
"At least one of AV1, VP9, H.265, H.264, or JPEG encoding must be enabled",
);
}
const canvas = document.createElement("canvas");
Expand Down Expand Up @@ -476,6 +508,21 @@ async function startVideoCaptureAsync(
encoders.push(encoder);
}
}
if (enableVP9) {
const encoder = await tryToConfigureEncoder({
outputFormat: "vp9",
baseConfig: BASE_CONFIG_VP9,
frameDurationSec,
framePool,
onError,
onVideoFrame,
signal,
video,
});
if (encoder) {
encoders.push(encoder);
}
}
if (enableAV1) {
const encoder = await tryToConfigureEncoder({
outputFormat: "av1",
Expand Down

0 comments on commit 212b873

Please sign in to comment.