diff --git a/src/chat-api/events/connectionEvents.ts b/src/chat-api/events/connectionEvents.ts index 43e6da89..8101753f 100644 --- a/src/chat-api/events/connectionEvents.ts +++ b/src/chat-api/events/connectionEvents.ts @@ -210,8 +210,8 @@ export const onAuthenticated = (payload: AuthenticatedPayload) => { } for (let i = 0; i < payload.voiceChannelUsers.length; i++) { - const voiceChannelUser = payload.voiceChannelUsers[i]; - voiceUsers.set(voiceChannelUser); + const voiceChannelUser = payload.voiceChannelUsers[i]!; + voiceUsers.createVoiceUser(voiceChannelUser); } }); diff --git a/src/chat-api/events/serverEvents.ts b/src/chat-api/events/serverEvents.ts index c8cd6945..8acc7437 100644 --- a/src/chat-api/events/serverEvents.ts +++ b/src/chat-api/events/serverEvents.ts @@ -60,8 +60,8 @@ export const onServerJoined = (payload: ServerJoinedPayload) => { users.setPresence(presence.userId, presence); } for (let i = 0; i < payload.voiceChannelUsers.length; i++) { - const rawVoice = payload.voiceChannelUsers[i]; - voiceUsers.set(rawVoice); + const rawVoice = payload.voiceChannelUsers[i]!; + voiceUsers.createVoiceUser(rawVoice); } }); }; @@ -75,7 +75,7 @@ export const onServerLeft = (payload: { serverId: string }) => const roles = useServerRoles(); const voiceUsers = useVoiceUsers(); - const currentVoiceChannelId = voiceUsers.currentVoiceChannelId(); + const currentVoiceChannelId = voiceUsers.currentUser()?.channelId; const serverChannels = channels.getChannelsByServerId(payload.serverId); @@ -90,7 +90,7 @@ export const onServerLeft = (payload: { serverId: string }) => const channel = serverChannels[i]!; account.removeNotificationSettings(channel.id); if (currentVoiceChannelId === channel.id) { - voiceUsers.setCurrentVoiceChannelId(null); + voiceUsers.setCurrentChannelId(null); } } }); diff --git a/src/chat-api/events/voiceEvents.ts b/src/chat-api/events/voiceEvents.ts index 91697f43..5ebaead3 100644 --- a/src/chat-api/events/voiceEvents.ts +++ b/src/chat-api/events/voiceEvents.ts @@ -4,19 +4,19 @@ import useAccount from "../store/useAccount"; import useVoiceUsers from "../store/useVoiceUsers"; export function onVoiceUserJoined(payload: RawVoice) { - const {set} = useVoiceUsers(); + const {createVoiceUser} = useVoiceUsers(); - set(payload); + createVoiceUser(payload); } export function onVoiceUserLeft(payload: {userId: string, channelId: string}) { - const {removeUserInVoice, setCurrentVoiceChannelId} = useVoiceUsers(); + const {removeVoiceUser, setCurrentChannelId} = useVoiceUsers(); const {user} = useAccount(); if (user()?.id === payload.userId) { - setCurrentVoiceChannelId(null); + setCurrentChannelId(null); } - removeUserInVoice(payload.channelId, payload.userId); + removeVoiceUser(payload.channelId, payload.userId); } interface VoiceSignalReceivedPayload { @@ -31,8 +31,8 @@ export function onVoiceSignalReceived(payload: VoiceSignalReceivedPayload) { if (!voiceUser) return; if (!voiceUser.peer) { - return voiceUser.addPeer(payload.signal); + return voiceUsers.createPeer(voiceUser, payload.signal); } - voiceUser.addSignal(payload.signal); + voiceUsers.signal(voiceUser, payload.signal); } \ No newline at end of file diff --git a/src/chat-api/services/VoiceService.ts b/src/chat-api/services/VoiceService.ts index 76f3ecfa..ef7d83ef 100644 --- a/src/chat-api/services/VoiceService.ts +++ b/src/chat-api/services/VoiceService.ts @@ -28,6 +28,9 @@ const lastCredentials = { generatedAt: null as null | number, result: null as null | any }; + + +export const getCachedCredentials = () => lastCredentials.result; export const postGenerateCredential = async () => { if (lastCredentials.generatedAt) { const diff = Date.now() - lastCredentials.generatedAt; diff --git a/src/chat-api/store/useAccount.ts b/src/chat-api/store/useAccount.ts index 8b7ca844..133feb83 100644 --- a/src/chat-api/store/useAccount.ts +++ b/src/chat-api/store/useAccount.ts @@ -114,6 +114,9 @@ const updateUserNotificationSettings = (opts: {serverId?: string, channelId?: st }; +const isMe = (userId: string) => account.user && account.user.id === userId; + + export default function useAccount() { return { user, @@ -129,6 +132,7 @@ export default function useAccount() { updateUserNotificationSettings, removeNotificationSettings, hasModeratorPerm, - lastAuthenticatedAt + lastAuthenticatedAt, + isMe }; } \ No newline at end of file diff --git a/src/chat-api/store/useChannels.ts b/src/chat-api/store/useChannels.ts index 15fe38a4..552becf6 100644 --- a/src/chat-api/store/useChannels.ts +++ b/src/chat-api/store/useChannels.ts @@ -109,17 +109,18 @@ function recipient(this: Channel) { return users.get(this.recipientId!); } + async function joinCall(this: Channel) { - const { setCurrentVoiceChannelId } = useVoiceUsers(); + const { setCurrentChannelId } = useVoiceUsers(); await postGenerateCredential(); postJoinVoice(this.id, socketClient.id()!).then(() => { - setCurrentVoiceChannelId(this.id); + setCurrentChannelId(this.id); }); } function leaveCall(this: Channel) { - const { setCurrentVoiceChannelId } = useVoiceUsers(); + const { setCurrentChannelId } = useVoiceUsers(); postLeaveVoice(this.id).then(() => { - setCurrentVoiceChannelId(null); + setCurrentChannelId(null); }); } function update(this: Channel, update: Partial) { @@ -154,7 +155,7 @@ const deleteChannel = (channelId: string, serverId?: string) => runWithContext(() => { const messages = useMessages(); const voice = useVoiceUsers(); - const voiceChannelId = voice.currentVoiceChannelId(); + const voiceChannelId = voice.currentUser(); if (serverId) { const servers = useServers(); @@ -172,7 +173,7 @@ const deleteChannel = (channelId: string, serverId?: string) => batch(() => { if (voiceChannelId && voiceChannelId === channelId) { - voice.setCurrentVoiceChannelId(null); + voice.setCurrentChannelId(null); } messages.deleteChannelMessages(channelId); diff --git a/src/chat-api/store/useServerMembers.ts b/src/chat-api/store/useServerMembers.ts index 031e4d06..d6f75b35 100644 --- a/src/chat-api/store/useServerMembers.ts +++ b/src/chat-api/store/useServerMembers.ts @@ -164,7 +164,7 @@ const remove = (serverId: string, userId: string) => { if (voiceChannelId) { const channel = channels.get(voiceChannelId); if (serverId === channel?.serverId) { - voiceUsers.removeUserInVoice(voiceChannelId, userId); + voiceUsers.removeVoiceUser(voiceChannelId, userId); } } diff --git a/src/chat-api/store/useVoiceUsers.ts b/src/chat-api/store/useVoiceUsers.ts index 3483fe75..c71a0a95 100644 --- a/src/chat-api/store/useVoiceUsers.ts +++ b/src/chat-api/store/useVoiceUsers.ts @@ -1,551 +1,522 @@ -import { batch, createSignal } from "solid-js"; import { createStore, reconcile } from "solid-js/store"; import { RawVoice } from "../RawData"; -import useUsers, { User } from "./useUsers"; +import { batch, createEffect, createSignal } from "solid-js"; +import { getCachedCredentials } from "../services/VoiceService"; +import { emitVoiceSignal } from "../emits/voiceEmits"; + import type SimplePeer from "@thaunknown/simple-peer"; +import useUsers, { User } from "./useUsers"; +import LazySimplePeer from "@/components/LazySimplePeer"; +import { getStorageString, StorageKeys } from "@/common/localStorage"; import useAccount from "./useAccount"; -import { emitVoiceSignal } from "../emits/voiceEmits"; -import useChannels from "./useChannels"; -import env from "@/common/env"; +import { set } from "idb-keyval"; import vad from "voice-activity-detection"; -import { getStorageString, StorageKeys } from "@/common/localStorage"; -import { postGenerateCredential } from "../services/VoiceService"; -interface VADInstance { - connect: () => void; - disconnect: () => void; - destroy: () => void; -} +const createIceServers = () => [ + getCachedCredentials(), + { + urls: ["stun:stun.l.google.com:19302"], + }, + { + urls: "stun:stun.relay.metered.ca:80", + }, + { + urls: "turn:a.relay.metered.ca:80", + username: "b9fafdffb3c428131bd9ae10", + credential: "DTk2mXfXv4kJYPvD", + }, + { + urls: "turn:a.relay.metered.ca:80?transport=tcp", + username: "b9fafdffb3c428131bd9ae10", + credential: "DTk2mXfXv4kJYPvD", + }, + { + urls: "turn:a.relay.metered.ca:443", + username: "b9fafdffb3c428131bd9ae10", + credential: "DTk2mXfXv4kJYPvD", + }, + { + urls: "turn:a.relay.metered.ca:443?transport=tcp", + username: "b9fafdffb3c428131bd9ae10", + credential: "DTk2mXfXv4kJYPvD", + }, +]; + +type StreamWithTracks = { + stream: MediaStream; + tracks: MediaStreamTrack[]; +}; + export type VoiceUser = RawVoice & { user: () => User; peer?: SimplePeer.Instance; - addSignal(this: VoiceUser, signal: SimplePeer.SignalData): void; - addPeer(this: VoiceUser, signal: SimplePeer.SignalData): void; - audioStream?: MediaStream; - videoStream?: MediaStream; - vad?: VADInstance; - voiceActivity: boolean; + streamWithTracks?: StreamWithTracks[]; audio?: HTMLAudioElement; - - waitingForVideoStreamId?: string; - waitingForAudioStreamId?: string; + voiceActivity?: boolean; + vadInstance?: ReturnType; }; +type ChannelUsersMap = Record; +type VoiceUsersMap = Record; + // voiceUsers[channelId][userId] = VoiceUser -const [voiceUsers, setVoiceUsers] = createStore< - Record> ->({}); -const [currentVoiceChannelId, _setCurrentVoiceChannelId] = createSignal< - null | string ->(null); - -interface LocalStreams { +const [voiceUsers, setVoiceUsers] = createStore({}); + +interface CurrentVoiceUser { + channelId: string; audioStream: MediaStream | null; videoStream: MediaStream | null; - vadStream: MediaStream | null; - vad: VADInstance | null; + vadInstance?: ReturnType; + vadAudioStream?: MediaStream | null; } -const [localStreams, setLocalStreams] = createStore({ - audioStream: null, - videoStream: null, - vad: null, - vadStream: null, -}); - -const set = async (voiceUser: RawVoice) => { - const users = useUsers(); - const account = useAccount(); - - if (!voiceUsers[voiceUser.channelId]) { - setVoiceUsers(voiceUser.channelId, {}); +const [currentVoiceUser, setCurrentVoiceUser] = createSignal< + CurrentVoiceUser | undefined +>(undefined); + +const setCurrentChannelId = (channelId: string | null) => { + const current = currentVoiceUser(); + if (current?.channelId) { + removeAllPeers(current?.channelId); + current.vadInstance?.destroy(); + current.vadAudioStream?.getAudioTracks()[0]?.stop(); } + if (!channelId) { + setCurrentVoiceUser(undefined); + return; + } + setCurrentVoiceUser({ + channelId, + audioStream: null, + videoStream: null, + micMuted: true, + }); +}; - let peer: SimplePeer.Instance | undefined; +const activeRemoteStream = (userId: string, kind: "audio" | "video") => { + const current = currentVoiceUser(); + if (!current) return; + const voiceUser = getVoiceUser(current.channelId, userId); + if (!voiceUser) return; - { - const isSelf = voiceUser.userId === account.user()?.id; - const isInVoice = currentVoiceChannelId() === voiceUser.channelId; + if (kind === "audio") { + return voiceUser.streamWithTracks?.find((stream) => + stream.tracks.every((track) => track.kind === kind) + )?.stream; + } else { + return voiceUser.streamWithTracks?.find((stream) => + stream.tracks.find((track) => track.kind === kind) + )?.stream; + } +}; - if (!isSelf && isInVoice) { - peer = await createPeer(voiceUser); +const removeAllPeers = (channelIdToRemove?: string) => { + batch(() => { + for (const channelId in voiceUsers) { + for (const userId in voiceUsers[channelId]) { + const voiceUser = getVoiceUser(channelId, userId); + if (!voiceUser) continue; + if (channelIdToRemove && voiceUser?.channelId !== channelIdToRemove) + continue; + voiceUser.peer?.destroy(); + voiceUser.vadInstance?.destroy(); + voiceUser.audio?.remove(); + setVoiceUsers(channelId, userId, "peer", undefined); + setVoiceUsers(channelId, userId, "streamWithTracks", []); + } } + }); +}; + +const getVoiceUsersByChannelId = (id: string) => { + return Object.values(voiceUsers[id] || {}) as VoiceUser[]; +}; + +const getVoiceUser = (channelId?: string, userId?: string) => { + return voiceUsers[channelId!]?.[userId!]; +}; +const removeVoiceUser = (channelId: string, userId: string) => { + const voiceUser = getVoiceUser(channelId, userId); + if (!voiceUser) return; + batch(() => { + voiceUser?.vadInstance?.destroy(); + voiceUser.peer?.destroy(); + voiceUser.audio?.remove(); + setVoiceUsers(channelId, userId, undefined); + }); +}; + +const createVoiceUser = (rawVoice: RawVoice) => { + const users = useUsers(); - const user = users.get(voiceUser.userId); - user.setVoiceChannelId(voiceUser.channelId); + if (!voiceUsers[rawVoice.channelId]) { + setVoiceUsers(rawVoice.channelId, {}); } - const newVoice: VoiceUser = { - ...voiceUser, - peer, - voiceActivity: false, + { + const user = users.get(rawVoice.userId); + user?.setVoiceChannelId(rawVoice.channelId); + } + + const newVoiceUser: VoiceUser = { + ...rawVoice, user, - addSignal, - addPeer, + streamWithTracks: [], }; - console.log("test"); - setVoiceUsers(voiceUser.channelId, voiceUser.userId, reconcile(newVoice)); + setVoiceUsers(rawVoice.channelId, rawVoice.userId, newVoiceUser); + + const isCurrentUserInVoice = + rawVoice.channelId === currentVoiceUser()?.channelId; + if (isCurrentUserInVoice) { + createPeer(newVoiceUser); + } }; -function addSignal(this: VoiceUser, signal: SimplePeer.SignalData) { - this.peer?.signal(signal); +function user(this: VoiceUser) { + const users = useUsers(); + return users.get(this.userId)!; } -async function addPeer(this: VoiceUser, signal: SimplePeer.SignalData) { - const user = this.user(); - console.log(user.username, "peer added"); +const createPeer = (voiceUser: VoiceUser, signal?: SimplePeer.SignalData) => { + const initiator = !signal; - const { default: LazySimplePeer } = await import("@thaunknown/simple-peer"); - const turnServer = await postGenerateCredential(); + const streams: MediaStream[] = []; + const current = currentVoiceUser(); + if (current?.audioStream) { + streams.push(current.audioStream); + } + if (current?.videoStream) { + streams.push(current.videoStream); + } const peer = new LazySimplePeer({ - initiator: false, + initiator, trickle: true, config: { - iceServers: [ - turnServer.result, - { - urls: ["stun:stun.l.google.com:19302"], - }, - { - urls: "stun:stun.relay.metered.ca:80", - }, - { - urls: "turn:a.relay.metered.ca:80", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:80?transport=tcp", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:443", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:443?transport=tcp", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - ], + iceServers: createIceServers(), }, - streams: [localStreams.audioStream, localStreams.videoStream].filter( - (stream) => stream - ) as MediaStream[], - }); - - peer.on("signal", (signal) => { - emitVoiceSignal(this.channelId, this.userId, signal); + streams, }); - peer.on("stream", (stream) => { - console.log("stream"); - onStream(this, stream); - }); + setVoiceUsers(voiceUser.channelId, voiceUser.userId, "peer", peer); - peer.on("data", (chunk: Uint8Array) => { - onData(this, Uint8ArrayToJson(chunk)); - }); peer.on("connect", () => { - console.log("connect"); - if (localStreams.audioStream) { - sendStreamToPeer(localStreams.audioStream, "audio"); - } - if (localStreams.videoStream) { - sendStreamToPeer(localStreams.videoStream, "video"); - } + console.log("RTC> Connected to", voiceUser.user().username + "!"); }); peer.on("end", () => { - console.log(user.username + " peer removed"); + console.log("RTC> Disconnected from", voiceUser.user().username + "."); + }); + peer.on("close", () => { + console.log("RTC>", voiceUser.user().username, "disconnected."); }); peer.on("error", (err) => { - console.log(err); + console.error(err); }); - peer.signal(signal); - - setVoiceUsers(this.channelId, this.userId, "peer", peer); -} - -function user(this: VoiceUser) { - const users = useUsers(); - return users.get(this.userId); -} - -const removeUserInVoice = (channelId: string, userId: string) => { - const voiceUser = voiceUsers[channelId][userId]; - batch(() => { - voiceUser?.vad?.destroy(); - voiceUser?.user().setVoiceChannelId(undefined); - voiceUser?.peer?.destroy(); - setVoiceUsers(channelId, userId, undefined); + peer.on("signal", (data) => { + emitVoiceSignal(voiceUser.channelId, voiceUser.userId, data); }); -}; -const getVoiceUsers = (channelId: string): VoiceUser[] => { - const account = useAccount(); - const selfUserId = account.user()?.id!; - return Object.values(voiceUsers[channelId] || {}).map((v) => { - if (v?.userId !== selfUserId) return v; - return { - ...v, - audioStream: localStreams.audioStream || undefined, - videoStream: localStreams.videoStream || undefined, + peer.on("track", (track, stream) => { + const channelId = voiceUser.channelId; + const userId = voiceUser.userId; + + stream.onremovetrack = (event) => { + const newVoiceUser = getVoiceUser(channelId, userId); + const activeAudioStream = activeRemoteStream(userId, "audio"); + if (activeAudioStream?.id === stream.id) { + newVoiceUser?.vadInstance?.destroy(); + setVoiceUsers(channelId, userId, { + voiceActivity: false, + vadInstance: undefined, + }); + } + + const streams = newVoiceUser?.streamWithTracks; + if (!streams) return; + const streamWithTracksIndex = streams.findIndex( + (s) => s.stream?.id === stream?.id + ); + const tracks = streams[streamWithTracksIndex]?.tracks; + + const newTracks = tracks?.filter((t) => t.id !== event.track.id); + if (!newTracks?.length) { + const newStreamWithTracks = streams.filter( + (s) => s.stream?.id !== stream?.id + ); + setVoiceUsers( + channelId, + userId, + "streamWithTracks", + newStreamWithTracks + ); + return; + } + + setVoiceUsers( + channelId, + userId, + "streamWithTracks", + streamWithTracksIndex, + "tracks", + newTracks + ); }; - }) as VoiceUser[]; -}; -const getVoiceUser = (channelId: string, userId: string) => { - return voiceUsers[channelId]?.[userId]; -}; + pushVoiceUserTrack(voiceUser, track, stream); -export async function createPeer(voiceUser: VoiceUser | RawVoice) { - const users = useUsers(); - const user = users.get(voiceUser.userId); - console.log(user.username, "peer created"); + const newVoiceUser = getVoiceUser(channelId, userId); - const { default: LazySimplePeer } = await import("@thaunknown/simple-peer"); - const turnServer = await postGenerateCredential(); + const streams = newVoiceUser?.streamWithTracks; + if (!streams) return; - const peer = new LazySimplePeer({ - initiator: true, - trickle: true, - config: { - iceServers: [ - turnServer.result, - { - urls: ["stun:stun.l.google.com:19302"], - }, - { - urls: "stun:stun.relay.metered.ca:80", - }, - { - urls: "turn:a.relay.metered.ca:80", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:80?transport=tcp", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:443", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - { - urls: "turn:a.relay.metered.ca:443?transport=tcp", - username: "b9fafdffb3c428131bd9ae10", - credential: "DTk2mXfXv4kJYPvD", - }, - ], - }, - streams: [localStreams.audioStream, localStreams.videoStream].filter( - (stream) => stream - ) as MediaStream[], - }); + const audio = newVoiceUser.audio || new Audio(); + const deviceId = getStorageString(StorageKeys.outputDeviceId, undefined); + if (deviceId) { + audio.setSinkId(JSON.parse(deviceId)); + } + const activeAudio = activeRemoteStream(userId, "audio"); - peer.on("signal", (signal) => { - emitVoiceSignal(voiceUser.channelId, voiceUser.userId, signal); - }); + newVoiceUser.vadInstance?.destroy(); - peer.on("stream", (stream) => { - console.log("stream"); - onStream(voiceUser, stream); - }); - peer.on("data", (chunk: Uint8Array) => { - onData(voiceUser, Uint8ArrayToJson(chunk)); - }); - peer.on("connect", () => { - console.log("connect"); - // if (localStreams.audioStream) { - // sendStreamToPeer(localStreams.audioStream, "audio"); - // } - // if (localStreams.videoStream) { - // sendStreamToPeer(localStreams.videoStream, "video"); - // } - }); - peer.on("end", () => { - console.log(user.username, "end"); - }); - peer.on("error", (err) => { - console.log(err); + const vadInstance = createVadInstance(activeAudio, undefined, userId); + batch(() => { + setVoiceUsers(channelId, userId, "vadInstance", vadInstance); + + audio.srcObject = activeAudio || null; + audio.play(); + if (!audio.srcObject) { + setVoiceUsers(channelId, userId, "audio", undefined); + } + setVoiceUsers(channelId, userId, "audio", audio); + }); }); - return peer; -} -function setLocalVAD(stream: MediaStream) { - const account = useAccount(); + if (signal) { + peer.signal(signal); + } +}; - const audioContext = new AudioContext(); - const track = localStreams.audioStream?.getAudioTracks()[0]!; - const vadInstance = vad(audioContext, stream, { - minNoiseLevel: 0.15, - noiseCaptureDuration: 0, +function createVadInstance( + vadStream?: MediaStream, + originalStream?: MediaStream, + userId?: string +) { + if (!vadStream) return; + const account = useAccount(); - onVoiceStart: function () { - setVoiceUsers(currentVoiceChannelId()!, account.user()?.id!, { - voiceActivity: true, - }); - track.enabled = true; - }, - onVoiceStop: function () { - setVoiceUsers(currentVoiceChannelId()!, account.user()?.id!, { - voiceActivity: false, - }); - track.enabled = false; - }, - }); - setLocalStreams({ vad: vadInstance }); -} + const originalStreamTrack = originalStream?.getAudioTracks()[0]; -function setVAD(stream: MediaStream, voiceUser: RawVoice) { + const current = currentVoiceUser(); + if (!current) return; const audioContext = new AudioContext(); - const vadInstance = vad(audioContext, stream, { - minNoiseLevel: 0, + const vadInstance = vad(audioContext, vadStream, { + ...(!userId + ? { + minNoiseLevel: 0.15, + noiseCaptureDuration: 0, + } + : { + minNoiseLevel: 0, + noiseCaptureDuration: 0, + avgNoiseMultiplier: 0.1, + }), - noiseCaptureDuration: 0, - avgNoiseMultiplier: 0.1, onVoiceStart: function () { - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { + setVoiceUsers(current.channelId, userId || account.user()?.id!, { voiceActivity: true, }); + if (originalStreamTrack) { + originalStreamTrack.enabled = true; + } }, onVoiceStop: function () { - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { + setVoiceUsers(current.channelId, userId || account.user()?.id!, { voiceActivity: false, }); + if (originalStreamTrack) { + originalStreamTrack.enabled = false; + } }, }); - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { vad: vadInstance }); + + return vadInstance; } -const onData = ( - rawVoice: RawVoice, - data?: { type: "video" | "audio"; streamId: string } +const pushVoiceUserTrack = ( + voiceUser: VoiceUser, + track: MediaStreamTrack, + stream: MediaStream ) => { - if (!data?.type || !data?.streamId) return; - const voiceUser = getVoiceUser(rawVoice.channelId, rawVoice.userId); - if (!voiceUser) return; - - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { - ...(data.type === "audio" - ? { waitingForAudioStreamId: data.streamId } - : {}), - ...(data.type === "video" - ? { waitingForVideoStreamId: data.streamId } - : {}), - }); -}; - -const onStream = (voiceUser: RawVoice, stream: MediaStream) => { - // const voiceUser = getVoiceUser(rawVoiceUser.channelId, rawVoiceUser.userId); - // if (!voiceUser) return; - // if (!voiceUser.waitingForAudioStreamId && !voiceUser.waitingForVideoStreamId) return; - - // const streamType = voiceUser.waitingForAudioStreamId === stream.id ? "audioStream" : "videoStream"; - - const videoTracks = stream.getVideoTracks(); - const streamType = videoTracks.length ? "videoStream" : "audioStream"; - - stream.onremovetrack = () => { - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { - [streamType]: null, - ...(streamType === "audioStream" - ? { audio: mic, voiceActivity: false } - : {}), - }); - stream.onremovetrack = null; - }; - - let mic: HTMLAudioElement | undefined = undefined; - if (streamType === "audioStream") { - setVAD(stream, voiceUser); - mic = new Audio(); - const deviceId = getStorageString(StorageKeys.outputDeviceId, undefined); - if (deviceId) { - mic.setSinkId(JSON.parse(deviceId)); - } - mic.srcObject = stream; - mic.play(); + const channelId = voiceUser.channelId; + const userId = voiceUser.userId; + + const newVoiceUser = getVoiceUser(channelId, userId); + + const streams = newVoiceUser?.streamWithTracks; + if (!streams) return; + + const streamWithTracksIndex = streams.findIndex( + (s) => s.stream.id === stream.id + ); + const streamWithTracks = streams[streamWithTracksIndex]; + + if (streamWithTracks && streamWithTracksIndex >= 0) { + setVoiceUsers( + channelId, + userId, + "streamWithTracks", + streamWithTracksIndex, + { + tracks: [...streamWithTracks.tracks, track], + } + ); + return; } - setVoiceUsers(voiceUser.channelId, voiceUser.userId, { - [streamType]: stream, - ...(streamType === "audioStream" ? { audio: mic } : {}), + + setVoiceUsers(channelId, userId, "streamWithTracks", streams.length, { + stream, + tracks: [track], }); }; +const toggleMic = async () => { + const userId = useAccount().user()?.id!; + const current = currentVoiceUser(); + if (!current) return; -const isLocalMicMuted = () => localStreams.audioStream === null; + if (current.audioStream) { + current.vadInstance?.destroy(); -const toggleMic = async () => { - const deviceId = getStorageString(StorageKeys.inputDeviceId, undefined); - if (isLocalMicMuted()) { - const stream = await navigator.mediaDevices.getUserMedia({ - audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) }, - video: false, + current.vadAudioStream?.getTracks().forEach((track) => { + track.stop(); }); - const vadStream = await navigator.mediaDevices.getUserMedia({ - audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) }, - video: false, + removeStream(current.audioStream); + setCurrentVoiceUser({ ...current, audioStream: null }); + setVoiceUsers(current.channelId, userId, { + voiceActivity: false, }); - setLocalStreams({ audioStream: stream, vadStream }); - setLocalVAD(vadStream); - sendStreamToPeer(stream, "audio"); return; } - const account = useAccount(); - localStreams.vadStream?.getAudioTracks()[0].stop(); - localStreams.vad?.destroy(); - stopStream(localStreams.audioStream!); - setLocalStreams({ audioStream: null }); - setVoiceUsers(currentVoiceChannelId()!, account.user()?.id!, { - voiceActivity: false, + const deviceId = getStorageString(StorageKeys.inputDeviceId, undefined); + const stream = await navigator.mediaDevices.getUserMedia({ + audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) }, + video: false, + }); + const vadStream = await navigator.mediaDevices.getUserMedia({ + audio: !deviceId ? true : { deviceId: JSON.parse(deviceId) }, + video: false, + }); + addStreamToPeers(stream); + const vadInstance = createVadInstance(vadStream, stream); + setCurrentVoiceUser({ + ...current, + audioStream: stream, + vadInstance, + vadAudioStream: vadStream, }); }; const setVideoStream = (stream: MediaStream | null) => { - if (localStreams.videoStream) { - localStreams.videoStream.getTracks().forEach((t) => t.stop()); - removeStreamFromPeer(localStreams.videoStream); + const current = currentVoiceUser(); + if (!current) return; + if (current.videoStream) { + removeStream(current.videoStream); } - setLocalStreams({ videoStream: stream }); + setCurrentVoiceUser({ ...current, videoStream: stream }); if (!stream) return; - sendStreamToPeer(stream, "video"); + addStreamToPeers(stream); - const videoTrack = stream.getVideoTracks()[0]; + const videoTrack = stream.getVideoTracks()[0]!; videoTrack.onended = () => { - stopStream(stream); - setLocalStreams({ videoStream: null }); + removeStream(stream); + setCurrentVoiceUser({ ...current, videoStream: null }); videoTrack.onended = null; }; }; -const micEnabled = (channelId: string, userId: string) => { - const account = useAccount(); - if (account.user()?.id === userId) { - return !!localStreams.audioStream; - } - return !!voiceUsers[channelId][userId]?.audioStream; -}; -const videoEnabled = (channelId: string, userId: string) => { - const account = useAccount(); - if (account.user()?.id === userId) { - return !!localStreams.videoStream; - } - return !!voiceUsers[channelId][userId]?.videoStream; +const removeStream = (stream: MediaStream) => { + removeStreamFromPeers(stream); + stream.getTracks().forEach((track) => { + track.stop(); + }); }; -const sendStreamToPeer = (stream: MediaStream, type: "audio" | "video") => { - console.log("sending stream..."); - - const voiceUsers = getVoiceUsers(currentVoiceChannelId()!); - for (let i = 0; i < voiceUsers.length; i++) { - const voiceUser = voiceUsers[i]; - // voiceUser?.peer?.write( - // jsonToUint8Array({ - // type, - // streamId: stream.id, - // }) - // ); - voiceUser?.peer?.addStream(stream); - } -}; -const removeStreamFromPeer = (stream: MediaStream) => { - console.log("removing stream..."); - const voiceUsers = getVoiceUsers(currentVoiceChannelId()!); - for (let i = 0; i < voiceUsers.length; i++) { - const voiceUser = voiceUsers[i]; - voiceUser?.peer?.removeStream(stream); - } -}; +const addStreamToPeers = (stream: MediaStream) => { + const current = currentVoiceUser(); + if (!current) return; + const voiceUsers = getVoiceUsersByChannelId(current.channelId); -const stopStream = (mediaStream: MediaStream) => { - removeStreamFromPeer(mediaStream); - mediaStream.getTracks().forEach((track) => track.stop()); + voiceUsers.forEach((voiceUser) => { + voiceUser.peer?.addStream(stream); + }); }; -const setCurrentVoiceChannelId = (channelId: string | null) => { - const channels = useChannels(); - - const oldChannelId = currentVoiceChannelId(); - if (oldChannelId) { - const channel = channels.get(oldChannelId); - channel?.setCallJoinedAt(undefined); - } - - const channel = channels.get(channelId!); - channel?.setCallJoinedAt(Date.now()); +const removeStreamFromPeers = (stream: MediaStream) => { + const current = currentVoiceUser(); + if (!current) return; + const voiceUsers = getVoiceUsersByChannelId(current.channelId); - const voiceUsers = getVoiceUsers(currentVoiceChannelId()!); - _setCurrentVoiceChannelId(channelId); + voiceUsers.forEach((voiceUser) => { + voiceUser.peer?.removeStream(stream); + }); +}; - localStreams.videoStream && stopStream(localStreams.videoStream); - if (localStreams.audioStream) { - localStreams.vadStream?.getAudioTracks()[0].stop(); - localStreams.vad?.destroy(); - stopStream(localStreams.audioStream); +const signal = (voiceUser: VoiceUser, signal: SimplePeer.SignalData) => { + if (!voiceUser.peer) { + console.error("No peer for voice user", voiceUser); + return; } - setLocalStreams({ videoStream: null, audioStream: null }); - if (!voiceUsers) return; - - batch(() => { - voiceUsers.forEach((voiceUser) => { - voiceUser?.peer?.destroy(); - voiceUser?.vad?.destroy(); - setVoiceUsers(voiceUser?.channelId!, voiceUser?.userId!, { - peer: undefined, - audioStream: undefined, - videoStream: undefined, - vad: undefined, - voiceActivity: false, - }); - }); - }); + voiceUser.peer.signal(signal); }; function resetAll() { batch(() => { - if (currentVoiceChannelId()) { - setCurrentVoiceChannelId(null); - } + removeAllPeers(); + setCurrentVoiceUser(undefined); setVoiceUsers(reconcile({})); }); } +const micEnabled = (userId: string) => { + const account = useAccount(); + if (account.user()?.id === userId) { + const currentUser = currentVoiceUser(); + return !!currentUser?.audioStream; + } + return activeRemoteStream(userId, "audio"); +}; + +const videoEnabled = (userId: string) => { + const account = useAccount(); + if (account.user()?.id === userId) { + const currentUser = currentVoiceUser(); + return currentUser?.videoStream; + } + return activeRemoteStream(userId, "video"); +}; + export default function useVoiceUsers() { return { - set, + createPeer, + createVoiceUser, getVoiceUser, - getVoiceUsers, - removeUserInVoice, - currentVoiceChannelId, - setCurrentVoiceChannelId, - isLocalMicMuted, - micEnabled, + getVoiceUsersByChannelId, + signal, + removeVoiceUser, + setCurrentChannelId, + currentUser: currentVoiceUser, + activeRemoteStream, videoEnabled, toggleMic, setVideoStream, resetAll, - localStreams, - }; -} -function jsonToUint8Array(json: T) { - return new TextEncoder().encode(JSON.stringify(json)); -} + isLocalMicMuted: () => !currentVoiceUser()?.audioStream, -function Uint8ArrayToJson(array: Uint8Array) { - try { - return JSON.parse(new TextDecoder().decode(array)); - } catch { - return null; - } + micEnabled, + }; } diff --git a/src/components/InVoiceActions.tsx b/src/components/InVoiceActions.tsx index ad4440b7..f666c12c 100644 --- a/src/components/InVoiceActions.tsx +++ b/src/components/InVoiceActions.tsx @@ -25,7 +25,7 @@ const DetailsContainer = styled(FlexColumn)` export default function InVoiceActions(props: { style?: JSX.CSSProperties }) { const { voiceUsers, channels, servers } = useStore(); - const channelId = () => voiceUsers.currentVoiceChannelId(); + const channelId = () => voiceUsers.currentUser()?.channelId; const channel = () => channels.get(channelId()!); const server = () => servers.get(channel()?.serverId!); diff --git a/src/components/LazySimplePeer.ts b/src/components/LazySimplePeer.ts new file mode 100644 index 00000000..52b346bd --- /dev/null +++ b/src/components/LazySimplePeer.ts @@ -0,0 +1,4 @@ + +const { default: LazySimplePeer } = await import("@thaunknown/simple-peer"); + +export default LazySimplePeer; \ No newline at end of file diff --git a/src/components/main-pane-header/MainPaneHeader.tsx b/src/components/main-pane-header/MainPaneHeader.tsx index a3a6808d..1fa19ef2 100644 --- a/src/components/main-pane-header/MainPaneHeader.tsx +++ b/src/components/main-pane-header/MainPaneHeader.tsx @@ -86,7 +86,7 @@ export default function MainPaneHeader() { }; const onCallClick = async () => { - if (voiceUsers.currentVoiceChannelId() === channel()?.id) return; + if (voiceUsers.currentUser()?.channelId === channel()?.id) return; channel()?.joinCall(); }; @@ -307,9 +307,9 @@ function VoiceHeader(props: { channelId?: string }) { const [selectedUserId, setSelectedUserId] = createSignal(null); - const channelVoiceUsers = () => voiceUsers.getVoiceUsers(props.channelId!); + const channelVoiceUsers = () => voiceUsers.getVoiceUsersByChannelId(props.channelId!); const videoStreamingUsers = () => - channelVoiceUsers().filter((v) => v.videoStream); + channelVoiceUsers().filter((v) => voiceUsers.videoEnabled(v.userId)); createEffect( on(videoStreamingUsers, (now, prev) => { @@ -326,8 +326,10 @@ function VoiceHeader(props: { channelId?: string }) { }; const isSomeoneVideoStreaming = () => - voiceUsers.videoEnabled(props.channelId!, account.user()?.id!) || - channelVoiceUsers().find((v) => v?.videoStream); + channelVoiceUsers().find((v) => voiceUsers.videoEnabled(v.userId)); + + + return ( @@ -348,7 +350,7 @@ function VoiceHeader(props: { channelId?: string }) { /> @@ -420,7 +422,7 @@ function VoiceParticipants(props: { }) { const { voiceUsers } = useStore(); - const channelVoiceUsers = () => voiceUsers.getVoiceUsers(props.channelId!); + const channelVoiceUsers = () => voiceUsers.getVoiceUsersByChannelId(props.channelId!); return (
@@ -453,16 +455,15 @@ function VoiceParticipantItem(props: { const isMuted = () => { return !voiceUsers.micEnabled( - props.voiceUser.channelId, props.voiceUser.userId ); }; const isVideoStreaming = () => - voiceUsers.videoEnabled(props.voiceUser.channelId, props.voiceUser.userId); + voiceUsers.videoEnabled(props.voiceUser.userId); const isInCall = () => - voiceUsers.currentVoiceChannelId() === props.voiceUser.channelId; + voiceUsers.currentUser()?.channelId === props.voiceUser.channelId; const talking = () => props.voiceUser.voiceActivity; const user = () => props.voiceUser.user()!; @@ -522,6 +523,8 @@ function VoiceActions(props: { channelId: string }) { const { createPortal } = useCustomPortal(); const { isMobileAgent } = useWindowProperties(); + const currentVoiceUser = () => voiceUsers.currentUser(); + const channel = () => channels.get(props.channelId); const onCallClick = async () => { @@ -532,7 +535,7 @@ function VoiceActions(props: { channelId: string }) { channel()?.leaveCall(); }; - const isInCall = () => voiceUsers.currentVoiceChannelId() === props.channelId; + const isInCall = () => voiceUsers.currentUser()?.channelId === props.channelId; const onScreenShareClick = () => { createPortal((close) => ); @@ -571,13 +574,13 @@ function VoiceActions(props: { channelId: string }) { /> - +