2025-11-04 20:24:15 +01:00
/*
2025-11-18 10:13:10 +01:00
Copyright 2025 Element Creations Ltd.
2025-11-04 20:24:15 +01:00
SPDX-License-IdFentifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
2025-11-07 17:36:16 -05:00
import {
type Participant ,
ParticipantEvent ,
2025-11-12 12:09:31 +01:00
type LocalParticipant ,
2025-11-21 16:14:12 +01:00
type ScreenShareCaptureOptions ,
2025-12-12 10:31:08 +01:00
RoomEvent ,
MediaDeviceFailure ,
2025-11-07 17:36:16 -05:00
} from "livekit-client" ;
import { observeParticipantEvents } from "@livekit/components-core" ;
2025-11-04 20:24:15 +01:00
import {
2025-12-02 19:40:08 +01:00
Status as RTCSessionStatus ,
2026-01-28 14:38:23 +01:00
type LivekitTransportConfig ,
2025-11-04 20:24:15 +01:00
type MatrixRTCSession ,
2026-01-28 16:33:52 +01:00
type Transport ,
2025-11-04 20:24:15 +01:00
} from "matrix-js-sdk/lib/matrixrtc" ;
import {
BehaviorSubject ,
2025-11-20 14:42:12 +01:00
catchError ,
2025-11-04 20:24:15 +01:00
combineLatest ,
2025-11-17 18:22:37 +01:00
distinctUntilChanged ,
2025-11-27 14:42:23 +01:00
from ,
2025-12-12 10:31:08 +01:00
fromEvent ,
2025-11-04 20:24:15 +01:00
map ,
type Observable ,
of ,
2025-12-02 19:40:08 +01:00
pairwise ,
2025-11-27 14:42:23 +01:00
startWith ,
2025-11-04 20:24:15 +01:00
switchMap ,
2025-11-14 16:18:31 +01:00
tap ,
2025-11-04 20:24:15 +01:00
} from "rxjs" ;
2025-11-14 16:18:31 +01:00
import { type Logger } from "matrix-js-sdk/lib/logger" ;
2025-11-25 20:18:34 +01:00
import { deepCompare } from "matrix-js-sdk/lib/utils" ;
2025-12-17 09:53:49 +01:00
import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager" ;
2025-11-04 20:24:15 +01:00
2025-12-16 13:40:06 +01:00
import { type Behavior } from "../../Behavior.ts" ;
2025-12-09 15:23:30 +01:00
import { type IConnectionManager } from "../remoteMembers/ConnectionManager.ts" ;
2026-01-15 19:04:32 +01:00
import { type ObservableScope } from "../../ObservableScope.ts" ;
2025-12-09 15:23:30 +01:00
import { type Publisher } from "./Publisher.ts" ;
import { type MuteStates } from "../../MuteStates.ts" ;
2025-11-25 20:18:34 +01:00
import {
ElementCallError ,
2025-12-10 18:50:19 +01:00
FailToStartLivekitConnection ,
2025-11-25 20:18:34 +01:00
MembershipManagerError ,
UnknownCallError ,
2025-12-09 15:23:30 +01:00
} from "../../../utils/errors.ts" ;
import { ElementWidgetActions , widget } from "../../../widget.ts" ;
2025-11-07 08:44:44 +01:00
import { getUrlParams } from "../../../UrlParams.ts" ;
2025-11-11 15:51:48 +01:00
import { PosthogAnalytics } from "../../../analytics/PosthogAnalytics.ts" ;
import { MatrixRTCMode } from "../../../settings/settings.ts" ;
import { Config } from "../../../config/Config.ts" ;
2025-12-02 19:40:08 +01:00
import {
2025-12-09 15:23:30 +01:00
ConnectionState ,
2025-12-02 19:40:08 +01:00
type Connection ,
2025-12-09 15:23:30 +01:00
type FailedToStartError ,
2025-12-02 19:40:08 +01:00
} from "../remoteMembers/Connection.ts" ;
import { type HomeserverConnected } from "./HomeserverConnected.ts" ;
2025-12-10 18:50:19 +01:00
import { and$ } from "../../../utils/observable.ts" ;
2026-01-09 13:38:26 +01:00
import { type LocalTransportWithSFUConfig } from "./LocalTransport.ts" ;
2025-11-14 16:18:31 +01:00
2025-12-09 15:23:30 +01:00
export enum TransportState {
2025-11-25 20:18:34 +01:00
/** Not even a transport is available to the LocalMembership */
2025-12-09 15:23:30 +01:00
Waiting = "transport_waiting" ,
}
export enum PublishState {
WaitingForUser = "publish_waiting_for_user" ,
2025-12-16 13:40:06 +01:00
// XXX: This state is removed for now since we do not have full control over
// track publication anymore with the publisher abstraction, might come back in the future?
2025-12-12 17:03:16 +01:00
// /** Implies lk connection is connected */
// Starting = "publish_start_publishing",
2025-12-02 19:40:08 +01:00
/** Implies lk connection is connected */
2025-12-09 15:23:30 +01:00
Publishing = "publish_publishing" ,
2025-11-04 20:24:15 +01:00
}
2025-11-14 16:18:31 +01:00
2025-12-12 14:40:45 +01:00
// TODO not sure how to map that correctly with the
// new publisher that does not manage tracks itself anymore
2025-12-09 15:23:30 +01:00
export enum TrackState {
/** The track is waiting for user input to create tracks (waiting to call `startTracks()`) */
WaitingForUser = "tracks_waiting_for_user" ,
2025-12-16 13:40:06 +01:00
// XXX: This state is removed for now since we do not have full control over
// track creation anymore with the publisher abstraction, might come back in the future?
2025-12-12 17:03:16 +01:00
// /** Implies lk connection is connected */
// Creating = "tracks_creating",
2025-12-09 15:23:30 +01:00
/** Implies lk connection is connected */
Ready = "tracks_ready" ,
2025-11-04 20:24:15 +01:00
}
2025-11-14 16:18:31 +01:00
2025-12-09 15:23:30 +01:00
export type LocalMemberMediaState =
| {
tracks : TrackState ;
connection : ConnectionState | FailedToStartError ;
}
| PublishState
| ElementCallError ;
export type LocalMemberState =
| ElementCallError
| TransportState . Waiting
| {
media : LocalMemberMediaState ;
2025-12-10 18:50:19 +01:00
matrix : ElementCallError | RTCSessionStatus ;
2025-12-09 15:23:30 +01:00
};
2025-11-05 12:56:58 +01:00
/*
* - get well known
* - get oldest membership
* - get transport to use
* - get openId + jwt token
* - wait for createTrack() call
* - create tracks
* - wait for join() call
* - Publisher.publishTracks()
* - send join state/sticky event
*/
2025-11-20 14:42:12 +01:00
2025-11-05 12:56:58 +01:00
interface Props {
2025-11-20 14:42:12 +01:00
// TODO add a comment into some code style readme or file header callviewmodel
// that the inputs for those createSomething$() functions should NOT contain any js-sdk objectes
2025-11-05 12:56:58 +01:00
scope : ObservableScope ;
muteStates : MuteStates ;
2025-11-06 21:54:34 +01:00
connectionManager : IConnectionManager ;
2026-01-15 19:04:32 +01:00
createPublisherFactory : ( connection : Connection ) => Publisher ;
2026-01-28 14:38:23 +01:00
joinMatrixRTC : ( transport : LivekitTransportConfig ) => void ;
2025-12-02 19:40:08 +01:00
homeserverConnected : HomeserverConnected ;
2026-01-09 13:38:26 +01:00
localTransport$ : Behavior < LocalTransportWithSFUConfig | null >;
2025-11-20 14:42:12 +01:00
matrixRTCSession : Pick <
MatrixRTCSession ,
"updateCallIntent" | "leaveRoomSession"
> ;
2025-11-14 16:18:31 +01:00
logger : Logger ;
2025-11-05 12:56:58 +01:00
}
2025-11-04 20:24:15 +01:00
/**
* This class is responsible for managing the own membership in a room.
* We want
* - a publisher
* -
2025-12-30 17:02:44 +01:00
* @param props The properties required to create the local membership.
* @param props.scope The observable scope to use.
* @param props.connectionManager The connection manager to get connections from.
* @param props.createPublisherFactory Factory to create a publisher once we have a connection.
* @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
* @param props.homeserverConnected The homeserver connected state.
* @param props.localTransport$ The local transport to use for publishing.
* @param props.logger The logger to use.
* @param props.muteStates The mute states for video and audio.
* @param props.matrixRTCSession The matrix RTC session to join.
2025-11-04 20:24:15 +01:00
* @returns
* - publisher: The handle to create tracks and publish them to the room.
2025-11-07 08:44:44 +01:00
* - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
2025-11-04 20:24:15 +01:00
* - transport$: the transport object the ownMembership$ ended up using.
2025-11-07 08:44:44 +01:00
* - connectionState: the current connection state. Including matrix server and livekit server connection.
* - sharingScreen$: Whether we are sharing our screen. `undefined` if we cannot share the screen.
2025-11-04 20:24:15 +01:00
*/
2025-11-05 18:57:24 +01:00
export const createLocalMembership$ = ({
2025-11-04 20:24:15 +01:00
scope ,
connectionManager ,
2025-11-20 14:42:12 +01:00
localTransport$ : localTransportCanThrow$ ,
2025-12-02 19:40:08 +01:00
homeserverConnected ,
2025-11-20 14:42:12 +01:00
createPublisherFactory ,
joinMatrixRTC ,
2025-11-14 16:18:31 +01:00
logger : parentLogger ,
2025-11-20 14:42:12 +01:00
muteStates ,
matrixRTCSession ,
2025-11-04 20:24:15 +01:00
} : Props ) : {
2025-12-01 14:42:15 +01:00
/**
2025-12-16 13:40:06 +01:00
* This request to start audio and video tracks.
* Can be called early to pre-emptively get media permissions and start devices.
2025-12-01 14:42:15 +01:00
*/
2025-12-16 13:40:06 +01:00
startTracks : () => void ;
2025-12-01 14:42:15 +01:00
/**
2025-12-09 15:23:30 +01:00
* This sets a inner state (shouldPublish) to true and instructs the js-sdk and livekit to keep the user
2025-12-01 14:42:15 +01:00
* connected to matrix and livekit.
*/
2025-12-09 15:23:30 +01:00
requestJoinAndPublish : () => void ;
2025-11-25 20:18:34 +01:00
requestDisconnect : () => void ;
2025-12-09 15:23:30 +01:00
localMemberState$ : Behavior < LocalMemberState >;
2025-11-12 15:02:19 -05:00
sharingScreen$ : Behavior < boolean >;
/**
* Callback to toggle screen sharing. If null, screen sharing is not possible.
*/
2025-11-07 08:44:44 +01:00
toggleScreenSharing : (() => void ) | null ;
2025-12-12 10:31:08 +01:00
// tracks$: Behavior<LocalTrack[]>;
2025-11-12 12:09:31 +01:00
participant$ : Behavior < LocalParticipant | null >;
connection$ : Behavior < Connection | null >;
2025-12-22 13:35:40 +01:00
/**
* Tracks the homserver and livekit connected state and based on that computes reconnecting.
2025-12-02 19:40:08 +01:00
*/
2025-11-05 12:56:58 +01:00
reconnecting$ : Behavior < boolean >;
2025-12-09 15:23:30 +01:00
/** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
2025-12-02 19:40:08 +01:00
* Direct translation to the js-sdk membership manager connection `Status`.
*/
disconnected$ : Behavior < boolean >;
2025-12-22 13:35:40 +01:00
/**
* Fully connected
*/
connected$ : Behavior < boolean >;
2026-01-19 12:10:18 +01:00
internalLoggerRef : Logger ;
2025-11-04 20:24:15 +01:00
} => {
2025-11-14 16:18:31 +01:00
const logger = parentLogger . getChild ( "[LocalMembership]" );
logger . debug ( `Creating local membership..` );
2025-11-04 20:24:15 +01:00
2025-11-25 20:18:34 +01:00
// Unwrap the local transport and set the state of the LocalMembership to error in case the transport is an error.
2025-11-20 14:42:12 +01:00
const localTransport$ = scope . behavior (
localTransportCanThrow$ . pipe (
catchError (( e : unknown ) => {
2025-11-21 13:04:28 +01:00
let error : ElementCallError ;
2025-11-20 14:42:12 +01:00
if ( e instanceof ElementCallError ) {
2025-11-21 13:04:28 +01:00
error = e ;
2025-11-20 14:42:12 +01:00
} else {
2025-11-21 13:04:28 +01:00
error = new UnknownCallError (
e instanceof Error
? e
: new Error ( "Unknown error from localTransport" ),
);
2025-11-20 14:42:12 +01:00
}
2025-12-09 15:23:30 +01:00
setTransportError ( error );
2025-11-20 14:42:12 +01:00
return of ( null );
}),
),
);
2025-11-06 21:54:34 +01:00
// Drop Epoch data here since we will not combine this anymore
2025-11-14 16:18:31 +01:00
const localConnection$ = scope . behavior (
2025-11-20 14:42:12 +01:00
combineLatest ([
connectionManager . connectionManagerData$ ,
localTransport$ ,
]). pipe (
2025-11-25 20:18:34 +01:00
map (([{ value : connectionData }, localTransport ]) => {
2025-11-14 16:18:31 +01:00
if ( localTransport === null ) {
return null ;
}
2025-11-20 14:42:12 +01:00
2026-01-09 13:38:26 +01:00
return connectionData . getConnectionForTransport (
localTransport . transport ,
);
2025-11-14 16:18:31 +01:00
}),
tap (( connection ) => {
logger . info (
`Local connection updated: ${ connection ? . transport ? . livekit_service_url } ` ,
);
}),
2025-11-04 20:24:15 +01:00
),
);
2025-12-12 10:31:08 +01:00
// Tracks error that happen when creating the local tracks.
const mediaErrors$ = localConnection$ . pipe (
switchMap (( connection ) => {
if ( ! connection ) {
return of ( null );
} else {
return fromEvent (
connection . livekitRoom ,
RoomEvent . MediaDevicesError ,
( error : Error ) => {
return MediaDeviceFailure . getFailure ( error ) ?? null ;
},
);
}
}),
);
mediaErrors$ . pipe ( scope . bind ()). subscribe (( error ) => {
if ( error ) {
2026-01-19 11:46:17 +01:00
// This is a MediaDevice error, can be PermissionDenied, NotFound, DeviceInUse, Other.
// Will also occurs if you cancel screen sharing browser prompt.
// This is not necessarily fatal, since the user might be able to join without media.
// XXX We might want to give some user feedback here to let them know their media is not working.
2025-12-12 10:31:08 +01:00
logger . error ( `Failed to create local tracks:` , error );
}
});
2025-11-04 20:24:15 +01:00
// MATRIX RELATED
2025-11-25 20:18:34 +01:00
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
2025-11-27 14:42:23 +01:00
const trackStartRequested = Promise . withResolvers < void >();
2025-11-25 20:18:34 +01:00
// This should be used in a combineLatest with publisher$ to connect.
// to make it possible to call startTracks before the preferredTransport$ has resolved.
2025-12-09 15:23:30 +01:00
const joinAndPublishRequested$ = new BehaviorSubject ( false );
2025-11-25 20:18:34 +01:00
/**
* The publisher is stored in here an abstracts creating and publishing tracks.
*/
const publisher$ = new BehaviorSubject < Publisher | null >( null );
2025-12-16 13:40:06 +01:00
const startTracks = () : void => {
2025-11-27 14:42:23 +01:00
trackStartRequested . resolve ();
2025-12-16 13:40:06 +01:00
// This used to return the tracks, but now they are only accessible via the publisher.
2025-11-04 20:24:15 +01:00
};
2025-12-09 15:23:30 +01:00
const requestJoinAndPublish = () : void => {
2025-11-27 14:42:23 +01:00
trackStartRequested . resolve ();
2025-12-09 15:23:30 +01:00
joinAndPublishRequested$ . next ( true );
2025-11-25 20:18:34 +01:00
};
const requestDisconnect = () : void => {
2025-12-09 15:23:30 +01:00
joinAndPublishRequested$ . next ( false );
2025-11-25 20:18:34 +01:00
};
// Take care of the publisher$
// create a new one as soon as a local Connection is available
//
// Recreate a new one once the local connection changes
// - stop publishing
// - destruct all current streams
// - overwrite current publisher
scope . reconcile ( localConnection$ , async ( connection ) => {
2026-01-15 18:11:43 +01:00
logger . info (
"reconcile based on new localConnection:" ,
connection ? . transport . livekit_service_url ,
);
2025-11-25 20:18:34 +01:00
if ( connection !== null ) {
2026-01-15 19:04:32 +01:00
const publisher = createPublisherFactory ( connection );
2025-12-08 23:33:41 -05:00
publisher$ . next ( publisher );
2026-01-15 18:11:43 +01:00
2026-01-15 19:04:32 +01:00
// Clean-up callback
2025-12-08 23:33:41 -05:00
return Promise . resolve ( async () : Promise < void > => {
2026-01-15 19:04:32 +01:00
await publisher . destroy ();
2025-12-08 23:33:41 -05:00
});
2025-11-04 20:24:15 +01:00
}
2025-11-07 19:07:45 +01:00
});
2025-11-12 12:09:31 +01:00
2025-11-25 20:18:34 +01:00
// Use reconcile here to not run concurrent createAndSetupTracks calls
// `tracks$` will update once they are ready.
scope . reconcile (
2025-11-27 14:42:23 +01:00
scope . behavior (
2025-12-12 10:31:08 +01:00
combineLatest ([
publisher$ /*, tracks$*/ ,
from ( trackStartRequested . promise ),
]),
2025-11-27 14:42:23 +01:00
null ,
),
async ( valueIfReady ) => {
if ( ! valueIfReady ) return ;
2025-12-12 10:31:08 +01:00
const [ publisher ] = valueIfReady ;
if ( publisher ) {
2025-11-25 20:18:34 +01:00
await publisher . createAndSetupTracks (). catch (( e ) => logger . error ( e ));
}
2025-11-07 19:07:45 +01:00
},
);
2025-11-25 20:18:34 +01:00
// Based on `connectRequested$` we start publishing tracks. (once they are there!)
scope . reconcile (
2025-12-12 14:40:45 +01:00
scope . behavior ( combineLatest ([ publisher$ , joinAndPublishRequested$ ])),
2025-12-12 11:54:43 +01:00
async ([ publisher , shouldJoinAndPublish ]) => {
2025-12-12 14:40:45 +01:00
// Get the current publishing state to avoid redundant calls.
const isPublishing = publisher ? . shouldPublish === true ;
if ( shouldJoinAndPublish && ! isPublishing ) {
2025-11-25 20:18:34 +01:00
try {
await publisher ? . startPublishing ();
} catch ( error ) {
2025-12-10 18:50:19 +01:00
const message =
error instanceof Error ? error.message : String ( error );
setPublishError ( new FailToStartLivekitConnection ( message ));
2025-11-25 20:18:34 +01:00
}
2025-12-12 14:40:45 +01:00
} else if ( isPublishing ) {
2025-11-25 20:18:34 +01:00
try {
await publisher ? . stopPublishing ();
} catch ( error ) {
2025-12-10 18:50:19 +01:00
setPublishError ( new UnknownCallError ( error as Error ));
2025-11-25 20:18:34 +01:00
}
}
},
);
2025-11-20 14:42:12 +01:00
2025-12-10 18:50:19 +01:00
// STATE COMPUTATION
// These are non fatal since we can join a room and concume media even though publishing failed.
const publishError$ = new BehaviorSubject < ElementCallError | null >( null );
const setPublishError = ( e : ElementCallError ) : void => {
2025-12-10 21:14:13 +01:00
if ( publishError$ . value !== null ) {
logger . error ( "Multiple Media Errors:" , e );
} else {
publishError$ . next ( e );
}
2025-12-09 15:23:30 +01:00
};
const fatalTransportError$ = new BehaviorSubject < ElementCallError | null >(
null ,
);
2025-12-10 21:14:13 +01:00
2025-12-09 15:23:30 +01:00
const setTransportError = ( e : ElementCallError ) : void => {
2025-12-10 21:14:13 +01:00
if ( fatalTransportError$ . value !== null ) {
2025-12-09 15:23:30 +01:00
logger . error ( "Multiple Transport Errors:" , e );
2025-12-10 21:14:13 +01:00
} else {
fatalTransportError$ . next ( e );
}
2025-11-25 20:18:34 +01:00
};
2025-12-09 15:23:30 +01:00
2025-12-10 18:50:19 +01:00
const localConnectionState$ = localConnection$ . pipe (
switchMap (( connection ) => ( connection ? connection.state$ : of ( null ))),
);
2025-12-09 15:23:30 +01:00
const mediaState$ : Behavior < LocalMemberMediaState > = scope . behavior (
2025-11-27 14:42:23 +01:00
combineLatest ([
2025-12-02 19:40:08 +01:00
localConnectionState$ ,
2025-11-27 14:42:23 +01:00
localTransport$ ,
2025-12-09 15:23:30 +01:00
joinAndPublishRequested$ ,
2025-11-27 14:42:23 +01:00
from ( trackStartRequested . promise ). pipe (
map (() => true ),
startWith ( false ),
),
]). pipe (
map (
([
2025-12-02 19:40:08 +01:00
localConnectionState ,
2025-11-27 14:42:23 +01:00
localTransport ,
2025-12-09 15:23:30 +01:00
shouldPublish ,
2025-11-27 14:42:23 +01:00
shouldStartTracks ,
]) => {
2025-12-09 15:23:30 +01:00
if ( ! localTransport ) return null ;
2025-12-12 14:40:45 +01:00
const trackState : TrackState = shouldStartTracks
? TrackState.Ready
: TrackState.WaitingForUser ;
2025-12-09 15:23:30 +01:00
2025-12-02 19:40:08 +01:00
if (
2025-12-09 15:23:30 +01:00
localConnectionState !== ConnectionState . LivekitConnected ||
trackState !== TrackState . Ready
2025-12-02 19:40:08 +01:00
)
2025-12-09 15:23:30 +01:00
return {
connection : localConnectionState ,
tracks : trackState ,
};
if ( ! shouldPublish ) return PublishState . WaitingForUser ;
2025-12-12 11:54:43 +01:00
// if (!publishing) return PublishState.Starting;
2025-12-09 15:23:30 +01:00
return PublishState . Publishing ;
2025-11-27 14:42:23 +01:00
},
),
distinctUntilChanged ( deepCompare ),
2025-11-25 20:18:34 +01:00
),
);
const fatalMatrixError$ = new BehaviorSubject < ElementCallError | null >( null );
const setMatrixError = ( e : ElementCallError ) : void => {
2025-12-10 21:14:13 +01:00
if ( fatalMatrixError$ . value !== null ) {
2025-11-25 20:18:34 +01:00
logger . error ( "Multiple Matrix Errors:" , e );
2025-12-10 21:14:13 +01:00
} else {
fatalMatrixError$ . next ( e );
}
2025-11-04 20:24:15 +01:00
};
2025-12-09 15:23:30 +01:00
const localMemberState$ = scope . behavior < LocalMemberState >(
combineLatest ([
mediaState$ ,
homeserverConnected . rtsSession$ ,
fatalMatrixError$ ,
fatalTransportError$ ,
2025-12-10 18:50:19 +01:00
publishError$ ,
2025-12-09 15:23:30 +01:00
]). pipe (
map (
([
mediaState ,
rtcSessionStatus ,
2025-12-10 18:50:19 +01:00
fatalMatrixError ,
fatalTransportError ,
publishError ,
2025-12-09 15:23:30 +01:00
]) => {
2025-12-10 18:50:19 +01:00
if ( fatalTransportError !== null ) return fatalTransportError ;
// `mediaState` will be 'null' until the transport/connection appears.
2025-12-09 15:23:30 +01:00
if ( mediaState && rtcSessionStatus )
return {
2025-12-10 18:50:19 +01:00
matrix : fatalMatrixError ?? rtcSessionStatus ,
media : publishError ?? mediaState ,
2025-12-09 15:23:30 +01:00
};
2025-12-10 12:08:59 +01:00
return TransportState . Waiting ;
2025-12-09 15:23:30 +01:00
},
),
2025-11-25 20:18:34 +01:00
),
);
2025-11-04 20:24:15 +01:00
2025-12-10 18:50:19 +01:00
/**
* Whether we are "fully" connected to the call. Accounts for both the
* connection to the MatrixRTC session and the LiveKit publish connection.
*/
const matrixAndLivekitConnected$ = scope . behavior (
and$ (
homeserverConnected . combined$ ,
localConnectionState$ . pipe (
map (( state ) => state === ConnectionState . LivekitConnected ),
),
). pipe (
tap (( v ) => logger . debug ( "livekit+matrix: Connected state changed" , v )),
),
);
/**
* Whether we should tell the user that we're reconnecting to the call.
*/
const reconnecting$ = scope . behavior (
matrixAndLivekitConnected$ . pipe (
pairwise (),
map (([ prev , current ]) => prev === true && current === false ),
),
2025-12-10 19:12:52 +01:00
false ,
2025-12-10 18:50:19 +01:00
);
2025-12-02 19:40:08 +01:00
// inform the widget about the connect and disconnect intent from the user.
scope
2025-12-09 15:23:30 +01:00
. behavior ( joinAndPublishRequested$ . pipe ( pairwise (), scope . bind ()), [
2025-12-02 19:40:08 +01:00
undefined ,
2025-12-09 15:23:30 +01:00
joinAndPublishRequested$ . value ,
2025-12-02 19:40:08 +01:00
])
. subscribe (([ prev , current ]) => {
if ( ! widget ) return ;
2025-12-10 12:08:59 +01:00
// JOIN prev=false (was left) => current-true (now joiend)
2025-12-02 19:40:08 +01:00
if ( ! prev && current ) {
2025-12-10 12:08:59 +01:00
widget . api . transport
. send ( ElementWidgetActions . JoinCall , {})
. catch (( e ) => {
logger . error ( "Failed to send join action" , e );
});
2025-12-02 19:40:08 +01:00
}
2025-12-10 12:08:59 +01:00
// LEAVE prev=false (was joined) => current-true (now left)
2025-12-02 19:40:08 +01:00
if ( prev && ! current ) {
2025-12-10 12:08:59 +01:00
widget . api . transport
. send ( ElementWidgetActions . HangupCall , {})
. catch (( e ) => {
logger . error ( "Failed to send hangup action" , e );
});
2025-12-02 19:40:08 +01:00
}
});
2026-01-19 12:10:18 +01:00
muteStates . video . enabled$ . pipe ( scope . bind ()). subscribe (( videoEnabled ) => {
void matrixRTCSession
. updateCallIntent ( videoEnabled ? "video" : "audio" )
. catch (( e ) => {
if ( e instanceof Error && e . message === "Not connected yet" ) {
logger . debug (
"'not connected yet' while updating the call intent (this is expected on startup)" ,
);
} else {
throw e ;
}
});
});
2025-12-02 19:40:08 +01:00
// Keep matrix rtc session in sync with localTransport$, connectRequested$
2025-11-25 20:18:34 +01:00
scope . reconcile (
2025-12-09 15:23:30 +01:00
scope . behavior ( combineLatest ([ localTransport$ , joinAndPublishRequested$ ])),
2025-11-25 20:18:34 +01:00
async ([ transport , shouldConnect ]) => {
2025-12-02 19:40:08 +01:00
if ( ! transport ) return ;
// if shouldConnect=false we will do the disconnect as the cleanup from the previous reconcile iteration.
2025-11-25 20:18:34 +01:00
if ( ! shouldConnect ) return ;
2025-11-04 20:24:15 +01:00
2025-11-25 20:18:34 +01:00
try {
2026-01-09 13:38:26 +01:00
joinMatrixRTC ( transport . transport );
2025-11-25 20:18:34 +01:00
} catch ( error ) {
logger . error ( "Error entering RTC session" , error );
if ( error instanceof Error )
setMatrixError ( new MembershipManagerError ( error ));
}
2025-12-02 19:40:08 +01:00
return Promise . resolve ( async () : Promise < void > => {
2025-11-25 20:18:34 +01:00
try {
2025-12-02 19:40:08 +01:00
// TODO Update matrixRTCSession to allow udpating the transport without leaving the session!
await matrixRTCSession . leaveRoomSession ( 1000 );
2025-11-25 20:18:34 +01:00
} catch ( e ) {
logger . error ( "Error leaving RTC session" , e );
}
2025-12-02 19:40:08 +01:00
});
2025-11-25 20:18:34 +01:00
},
);
const participant$ = scope . behavior (
2026-01-08 12:27:17 +01:00
localConnection$ . pipe (
map (( c ) => c ? . livekitRoom ? . localParticipant ?? null ),
tap (( p ) => {
logger . debug ( "participant$ updated:" , p ? . identity );
}),
),
2025-11-25 20:18:34 +01:00
);
2025-11-04 20:24:15 +01:00
2025-11-05 12:56:58 +01:00
// Pause upstream of all local media tracks when we're disconnected from
// MatrixRTC, because it can be an unpleasant surprise for the app to say
// 'reconnecting' and yet still be transmitting your media to others.
// We use matrixConnected$ rather than reconnecting$ because we want to
// pause tracks during the initial joining sequence too until we're sure
// that our own media is displayed on screen.
2025-11-25 20:18:34 +01:00
// TODO refactor this based no livekitState$
2025-12-02 19:40:08 +01:00
combineLatest ([ participant$ , homeserverConnected . combined$ ])
2025-11-05 12:56:58 +01:00
. pipe ( scope . bind ())
2025-11-25 20:18:34 +01:00
. subscribe (([ participant , connected ]) => {
if ( ! participant ) return ;
const publications = participant . trackPublications . values ();
2025-11-05 12:56:58 +01:00
if ( connected ) {
for ( const p of publications ) {
if ( p . track ? . isUpstreamPaused === true ) {
const kind = p . track . kind ;
2025-11-14 10:44:16 +01:00
logger . info (
`Resuming ${ kind } track (MatrixRTC connection present)` ,
);
2025-11-05 12:56:58 +01:00
p . track
. resumeUpstream ()
. catch (( e ) =>
logger . error (
`Failed to resume ${ kind } track after MatrixRTC reconnection` ,
e ,
),
);
}
}
} else {
for ( const p of publications ) {
if ( p . track ? . isUpstreamPaused === false ) {
const kind = p . track . kind ;
2025-11-14 10:44:16 +01:00
logger . info (
2025-11-05 12:56:58 +01:00
`Pausing ${ kind } track (uncertain MatrixRTC connection)` ,
);
p . track
. pauseUpstream ()
. catch (( e ) =>
logger . error (
`Failed to pause ${ kind } track after entering uncertain MatrixRTC connection` ,
e ,
),
);
}
}
}
});
2025-11-17 18:22:37 +01:00
2025-11-07 08:44:44 +01:00
/**
2025-11-12 15:02:19 -05:00
* Whether the user is currently sharing their screen.
2025-11-07 08:44:44 +01:00
*/
const sharingScreen$ = scope . behavior (
2025-11-25 20:18:34 +01:00
participant$ . pipe (
switchMap (( p ) => ( p !== null ? observeSharingScreen$ ( p ) : of ( false ))),
2025-11-07 08:44:44 +01:00
),
);
2025-12-01 14:42:15 +01:00
let toggleScreenSharing : (() => void ) | null = null ;
2025-11-21 16:14:12 +01:00
if (
2025-11-07 08:44:44 +01:00
"getDisplayMedia" in ( navigator . mediaDevices ?? {}) &&
! getUrlParams (). hideScreensharing
2025-11-21 16:14:12 +01:00
) {
toggleScreenSharing = () : void => {
const screenshareSettings : ScreenShareCaptureOptions = {
audio : true ,
selfBrowserSurface : "include" ,
surfaceSwitching : "include" ,
systemAudio : "include" ,
};
const targetScreenshareState = ! sharingScreen$ . value ;
logger . info (
`toggleScreenSharing called. Switching ${
targetScreenshareState ? "On" : "Off"
} ` ,
);
// If a connection is ready, toggle screen sharing.
// We deliberately do nothing in the case of a null connection because
// it looks nice for the call control buttons to all become available
// at once upon joining the call, rather than introducing a disabled
// state. The user can just click again.
// We also allow screen sharing to be toggled even if the connection
// is still initializing or publishing tracks, because there's no
// technical reason to disallow this. LiveKit will publish if it can.
2025-11-25 20:18:34 +01:00
participant$ . value
? . setScreenShareEnabled ( targetScreenshareState , screenshareSettings )
2025-11-21 16:14:12 +01:00
. catch ( logger . error );
};
}
2025-11-07 08:44:44 +01:00
2025-11-04 20:24:15 +01:00
return {
startTracks ,
2025-12-09 15:23:30 +01:00
requestJoinAndPublish ,
2025-11-04 20:24:15 +01:00
requestDisconnect ,
2025-12-09 15:23:30 +01:00
localMemberState$ ,
2025-11-27 14:42:23 +01:00
participant$ ,
2025-12-10 18:50:19 +01:00
reconnecting$ ,
2025-12-22 13:35:40 +01:00
connected$ : matrixAndLivekitConnected$ ,
2025-12-02 19:40:08 +01:00
disconnected$ : scope.behavior (
2025-12-09 15:23:30 +01:00
homeserverConnected . rtsSession$ . pipe (
map (( state ) => state === RTCSessionStatus . Disconnected ),
2025-12-02 19:40:08 +01:00
),
),
2025-11-07 08:44:44 +01:00
sharingScreen$ ,
toggleScreenSharing ,
2025-11-14 16:18:31 +01:00
connection$ : localConnection$ ,
2026-01-19 12:10:18 +01:00
internalLoggerRef : logger ,
2025-11-04 20:24:15 +01:00
};
};
2025-11-07 17:36:16 -05:00
export function observeSharingScreen$ ( p : Participant ) : Observable < boolean > {
return observeParticipantEvents (
p ,
ParticipantEvent . TrackPublished ,
ParticipantEvent . TrackUnpublished ,
ParticipantEvent . LocalTrackPublished ,
ParticipantEvent . LocalTrackUnpublished ,
). pipe ( map (( p ) => p . isScreenShareEnabled ));
}
2025-11-11 15:51:48 +01:00
interface EnterRTCSessionOptions {
encryptMedia : boolean ;
matrixRTCMode : MatrixRTCMode ;
}
/**
2025-11-18 12:14:17 +01:00
* Does the necessary steps to enter the RTC session on the matrix side:
* - Preparing the membership info (FOCUS to use, options)
* - Sends the matrix event to join the call, and starts the membership manager:
* - Delay events management
* - Handles retries (fails only after several attempts)
*
2025-12-30 17:02:44 +01:00
* @param rtcSession - The MatrixRTCSession to join.
2026-01-05 22:20:19 +01:00
* @param ownMembershipIdentity - Options for entering the RTC session.
2025-12-30 17:02:44 +01:00
* @param transport - The LivekitTransport to use for this session.
2026-01-05 22:20:19 +01:00
* @param options - `encryptMedia`: Whether to encrypt media `matrixRTCMode`: The Matrix RTC mode to use.
2025-11-14 16:18:31 +01:00
* @throws If the widget could not send ElementWidgetActions.JoinCall action.
2025-11-11 15:51:48 +01:00
*/
2025-11-14 10:48:24 -05:00
// Exported for unit testing
2025-12-02 19:40:08 +01:00
export function enterRTCSession (
2025-11-11 15:51:48 +01:00
rtcSession : MatrixRTCSession ,
2025-12-17 09:53:49 +01:00
ownMembershipIdentity : CallMembershipIdentityParts ,
2026-01-28 14:38:23 +01:00
transport : LivekitTransportConfig ,
2026-01-05 22:20:19 +01:00
options : EnterRTCSessionOptions ,
2025-12-02 19:40:08 +01:00
) : void {
2026-01-05 22:20:19 +01:00
const { encryptMedia , matrixRTCMode } = options ;
2025-11-11 15:51:48 +01:00
PosthogAnalytics . instance . eventCallEnded . cacheStartCall ( new Date ());
PosthogAnalytics . instance . eventCallStarted . track ( rtcSession . room . roomId );
// This must be called before we start trying to join the call, as we need to
// have started tracking by the time calls start getting created.
// groupCallOTelMembership?.onJoinCall();
const { features , matrix_rtc_session : matrixRtcSessionConfig } = Config . get ();
const useDeviceSessionMemberEvents =
features ? . feature_use_device_session_member_events ;
const { sendNotificationType : notificationType , callIntent } = getUrlParams ();
2026-01-07 16:08:20 +01:00
const multiSFU =
matrixRTCMode === MatrixRTCMode . Compatibility ||
matrixRTCMode === MatrixRTCMode . Matrix_2_0 ;
2026-01-28 16:33:52 +01:00
// For backwards compatibility with Element Call versions that do not do Matrix 2.0,
// we add the livekit alias to the transport.
let backwardCompatibleTransport : Transport ;
if ( matrixRTCMode === MatrixRTCMode . Matrix_2_0 ) {
backwardCompatibleTransport = transport ;
} else {
backwardCompatibleTransport = {
livekit_alias : rtcSession.room.roomId ,
... transport ,
};
}
2025-11-11 15:51:48 +01:00
// Multi-sfu does not need a preferred foci list. just the focus that is actually used.
2025-11-20 14:42:12 +01:00
// TODO where/how do we track errors originating from the ongoing rtcSession?
2026-01-28 16:33:52 +01:00
2025-12-17 09:53:49 +01:00
rtcSession . joinRTCSession (
ownMembershipIdentity ,
2026-01-28 16:33:52 +01:00
multiSFU ? [] : [ backwardCompatibleTransport ],
2026-01-28 16:42:07 +01:00
multiSFU ? backwardCompatibleTransport : undefined ,
2025-11-11 15:51:48 +01:00
{
notificationType ,
callIntent ,
manageMediaKeys : encryptMedia ,
...( useDeviceSessionMemberEvents !== undefined && {
useLegacyMemberEvents : ! useDeviceSessionMemberEvents ,
}),
delayedLeaveEventRestartMs :
matrixRtcSessionConfig?.delayed_leave_event_restart_ms ,
delayedLeaveEventDelayMs :
matrixRtcSessionConfig?.delayed_leave_event_delay_ms ,
delayedLeaveEventRestartLocalTimeoutMs :
matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms ,
networkErrorRetryMs : matrixRtcSessionConfig?.network_error_retry_ms ,
makeKeyDelay : matrixRtcSessionConfig?.wait_for_key_rotation_ms ,
membershipEventExpiryMs :
matrixRtcSessionConfig?.membership_event_expiry_ms ,
useExperimentalToDeviceTransport : true ,
unstableSendStickyEvents : matrixRTCMode === MatrixRTCMode . Matrix_2_0 ,
},
);
}