From 6aad9c50a4d656e5cdaf73d6843d8a2479c2d78b Mon Sep 17 00:00:00 2001 From: Angelika Serwa Date: Wed, 30 Oct 2024 12:15:54 +0100 Subject: [PATCH] [FCE-555] Add ex-webrtc support (#177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description This PR adds support for ex-webrtc backend. On both platforms: - ~removed `restartIce()` call - it causes issues with connectivity~ => `restartIce` is needed for non exwebrtc. - changes ice transport type from `.relay` to .`all` - conforms to protocol when sending candidates - disables simulcast in example app - it's currently not supported, but maybe it will be supported in the future, thus I'm not changing the api, it's disabled by default anyway NEW changes: - Removed `endpointType` as it was unused and only created confusion about what should be set. - ExWebRTC/WebRTC is now decided based upon if turn servers are passed (same as it's done on Web) On Android: - fixes a (backend) crash when client sends candidate before sending sdp offer by queueing the candidates - on iOS it's not needed as sdp offer is created on the same thread as candidates. In my opinion it's more of a backend issue but it's quicker to workaround it on our side. - fixes a bug when using `changeWebRTCLoggingSeverity` (found out when debugging something) - changes number strategy when serializing to json, so that it sends integers instead of floats On iOS: - removed some dead code with `queuedRemoteCandidates` ## Motivation and Context https://linear.app/swmansion/issue/FCE-555/investigate-work-required-to-support-elixirwebrtc-in-mobile-app ## How has this been tested? - Get a backend with ex-webrtc (built locally from `ex_webrtc_experimental` of fishjam or deployed to fishjam.work) - Open fishjam dashboard - Test only on physical devices, emulators won't work - currently backend has fixed h264 encoding which is unsupported on android emulator - Test different combinations: two mobiles, mobile + web - Test connecting in different order, screencast, disconnecting and reconnecting, adding and removing tracks --------- Co-authored-by: Miłosz Filimowski Co-authored-by: Milosz Filimowski --- .../screens/PreviewScreen/PreviewScreen.tsx | 2 +- .../client/FishjamClientInternal.kt | 21 ++--- .../com/fishjamcloud/client/events/Event.kt | 14 +-- .../fishjamcloud/client/models/Endpoint.kt | 1 - .../client/models/EndpointType.kt | 9 -- .../client/webrtc/PeerConnectionManager.kt | 86 ++++++++++++++----- .../client/webrtc/RTCEngineCommunication.kt | 10 ++- .../client/webrtc/RTCEngineListener.kt | 4 +- .../FishjamClient/FishjamClientInternal.swift | 19 ++-- .../Sources/FishjamClient/events/Event.swift | 8 +- .../FishjamClient/models/Endpoint.swift | 27 +----- .../webrtc/PeerConnectionManager.swift | 31 +++---- .../webrtc/RTCEngineCommunication.swift | 12 ++- .../webrtc/RTCEngineListener.swift | 2 +- .../io/fishjam/reactnative/RNFishjamClient.kt | 1 - .../reactnative/RNFishjamClientModule.kt | 2 +- .../ios/RNFishjamClient.swift | 1 - 17 files changed, 135 insertions(+), 115 deletions(-) delete mode 100644 packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt diff --git a/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx b/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx index f657e5f8..0b1f82e1 100644 --- a/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx +++ b/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx @@ -64,7 +64,7 @@ function PreviewScreen({ useEffect(() => { prepareCamera({ - simulcastEnabled: true, + simulcastEnabled: false, quality: 'HD169', cameraEnabled: true, }); diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt index c497cf04..cf298c59 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt @@ -13,7 +13,6 @@ import com.fishjamcloud.client.media.Track import com.fishjamcloud.client.models.AuthError import com.fishjamcloud.client.models.EncodingReason import com.fishjamcloud.client.models.Endpoint -import com.fishjamcloud.client.models.EndpointType import com.fishjamcloud.client.models.Metadata import com.fishjamcloud.client.models.Peer import com.fishjamcloud.client.models.RTCStats @@ -60,7 +59,7 @@ internal class FishjamClientInternal( private val commandsQueue: CommandsQueue = CommandsQueue() private var webSocket: WebSocket? = null - private var localEndpoint: Endpoint = Endpoint(id = "", type = EndpointType.WEBRTC) + private var localEndpoint: Endpoint = Endpoint(id = "") private var prevTracks = mutableListOf() private var remoteEndpoints: MutableMap = mutableMapOf() @@ -203,7 +202,7 @@ internal class FishjamClientInternal( webSocket = null remoteEndpoints = mutableMapOf() prevTracks = localEndpoint.tracks.values.toMutableList() - localEndpoint = Endpoint(id = "", type = EndpointType.WEBRTC) + localEndpoint = Endpoint(id = "") } private fun join() { @@ -224,7 +223,7 @@ internal class FishjamClientInternal( localEndpoint = localEndpoint.copy(id = endpointID) otherEndpoints.forEach { - var endpoint = Endpoint(it.id, EndpointType.fromString(it.type), it.metadata) + var endpoint = Endpoint(it.id, it.metadata) for ((trackId, trackData) in it.tracks) { val track = Track(null, it.id, trackId, trackData.metadata ?: mapOf()) @@ -258,7 +257,7 @@ internal class FishjamClientInternal( rtcEngineCommunication.disconnect() localEndpoint.tracks.values.forEach { (it as? LocalTrack)?.stop() } peerConnectionManager.close() - localEndpoint = Endpoint(id = "", type = EndpointType.WEBRTC) + localEndpoint = Endpoint(id = "") remoteEndpoints = mutableMapOf() peerConnectionManager.removeListener(this@FishjamClientInternal) rtcEngineCommunication.removeListener(this@FishjamClientInternal) @@ -552,14 +551,13 @@ internal class FishjamClientInternal( override fun onEndpointAdded( endpointId: String, - type: EndpointType, metadata: Metadata? ) { if (endpointId == this.localEndpoint.id) { return } - val endpoint = Endpoint(endpointId, type, metadata) + val endpoint = Endpoint(endpointId, metadata) remoteEndpoints[endpoint.id] = endpoint @@ -639,6 +637,7 @@ internal class FishjamClientInternal( localEndpoint.tracks.map { (_, track) -> track.webrtcId() to track.metadata }.toMap(), offer.midToTrackIdMapping ) + peerConnectionManager.onSentSdpOffer() } catch (e: Exception) { Timber.e(e, "Failed to create an sdp offer") } @@ -648,12 +647,12 @@ internal class FishjamClientInternal( override fun onRemoteCandidate( candidate: String, sdpMLineIndex: Int, - sdpMid: String? + sdpMid: Int? ) { coroutineScope.launch { val iceCandidate = IceCandidate( - sdpMid ?: "", + sdpMid?.toString() ?: "", sdpMLineIndex, candidate ) @@ -816,7 +815,9 @@ internal class FishjamClientInternal( override fun onLocalIceCandidate(candidate: IceCandidate) { coroutineScope.launch { - rtcEngineCommunication.localCandidate(candidate.sdp, candidate.sdpMLineIndex) + val splitSdp = candidate.sdp.split(" ") + val ufrag = splitSdp[splitSdp.indexOf("ufrag") + 1] + rtcEngineCommunication.localCandidate(candidate.sdp, candidate.sdpMLineIndex, candidate.sdpMid.toInt(), ufrag) } } } diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt index 4d76cfca..33eab18b 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt @@ -3,13 +3,14 @@ package com.fishjamcloud.client.events import com.fishjamcloud.client.models.Metadata import com.fishjamcloud.client.models.Payload import com.fishjamcloud.client.models.SimulcastConfig -import com.google.gson.Gson +import com.google.gson.GsonBuilder import com.google.gson.JsonParseException +import com.google.gson.ToNumberPolicy import com.google.gson.annotations.SerializedName import com.google.gson.reflect.TypeToken import timber.log.Timber -internal val gson = Gson() +internal val gson = GsonBuilder().setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create() // convert a data class to a map internal fun T.serializeToMap(): Map = convert() @@ -63,7 +64,7 @@ data class LocalCandidate( val type: String, val data: Payload ) : SendableEvent() { - constructor(candidate: String, sdpMLineIndex: Int) : + constructor(candidate: String, sdpMLineIndex: Int, sdpMid: Int?, usernameFragment: String?) : this( "custom", mapOf( @@ -71,7 +72,9 @@ data class LocalCandidate( "data" to mapOf( "candidate" to candidate, - "sdpMLineIndex" to sdpMLineIndex + "sdpMLineIndex" to sdpMLineIndex, + "sdpMid" to sdpMid, + "usernameFragment" to usernameFragment ) ) ) @@ -375,7 +378,8 @@ data class RemoteCandidate( data class Data( val candidate: String, val sdpMLineIndex: Int, - val sdpMid: String? + val sdpMid: Int?, + val usernameFragment: String? ) } diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt index 56d20f0d..43f530af 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt @@ -4,7 +4,6 @@ import com.fishjamcloud.client.media.Track data class Endpoint( val id: String, - val type: EndpointType, val metadata: Metadata? = mapOf(), val tracks: Map = mapOf() ) { diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt deleted file mode 100644 index ee143be6..00000000 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.fishjamcloud.client.models - -enum class EndpointType { - WEBRTC; - - companion object { - fun fromString(type: String): EndpointType = EndpointType.valueOf(type.uppercase()) - } -} diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt index 807a26d4..8816f069 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt @@ -57,6 +57,13 @@ internal class PeerConnectionManager( private var streamIds: List = listOf(UUID.randomUUID().toString()) + // ex-webrtc backend crashes if it's sent a candidate before sdp offer + // we can get local candidates from peer connection any time on a different thread + // so we queue them here and send them after sending sdp offer + private var sentSdpOffer = false + private var queuedLocalCandidates = mutableListOf() + private var qlcMutext = Mutex() + private fun getSendEncodingsFromConfig(simulcastConfig: SimulcastConfig): List { val sendEncodings = Constants.simulcastEncodings() simulcastConfig.activeEncodings.forEach { @@ -263,28 +270,32 @@ internal class PeerConnectionManager( return } - this.iceServers = - integratedTurnServers.map { - val url = - listOf( - "turn", - ":", - it.serverAddr, - ":", - it.serverPort.toString(), - "?transport=", - it.transport - ).joinToString("") - - PeerConnection.IceServer - .builder(url) - .setUsername(it.username) - .setPassword(it.password) - .createIceServer() - } + val isExWebrtc = integratedTurnServers.isEmpty() + + if (isExWebrtc) { + val iceServerList = listOf("stun:stun.l.google.com:19302", "stun:stun.l.google.com:5349") + this.iceServers = listOf(PeerConnection.IceServer.builder(iceServerList).createIceServer()) + } else { + this.iceServers = + integratedTurnServers.map { + val url = "turn:${it.serverAddr}:${it.serverPort}?transport=${it.transport}" + + PeerConnection.IceServer + .builder(url) + .setUsername(it.username) + .setPassword(it.password) + .createIceServer() + } + } val config = PeerConnection.RTCConfiguration(iceServers) - config.iceTransportsType = PeerConnection.IceTransportsType.RELAY + config.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN + config.iceTransportsType = + if (isExWebrtc) { + PeerConnection.IceTransportsType.ALL + } else { + PeerConnection.IceTransportsType.RELAY + } this.config = config } @@ -369,9 +380,15 @@ internal class PeerConnectionManager( tracksTypes: Map, localTracks: List ): SdpOffer { + val isExWebrtc = integratedTurnServers.isEmpty() + qrcMutex.withLock { this@PeerConnectionManager.queuedRemoteCandidates = mutableListOf() } + qlcMutext.withLock { + sentSdpOffer = false + this.queuedLocalCandidates = mutableListOf() + } prepareIceServers(integratedTurnServers) var needsRestart = true @@ -379,10 +396,11 @@ internal class PeerConnectionManager( setupPeerConnection(localTracks) needsRestart = false } + peerConnectionMutex.withLock { val pc = peerConnection!! - if (needsRestart) { + if (needsRestart && !isExWebrtc) { pc.restartIce() } @@ -475,9 +493,31 @@ internal class PeerConnectionManager( Timber.d("Change ice gathering state to $state") } + suspend fun onSentSdpOffer() { + qlcMutext.withLock { + sentSdpOffer = true + queuedLocalCandidates.forEach { candidate -> + listeners.forEach { listener -> + listener.onLocalIceCandidate( + candidate + ) + } + } + queuedLocalCandidates = mutableListOf() + } + } + override fun onIceCandidate(candidate: IceCandidate?) { - if (candidate != null) { - listeners.forEach { listener -> listener.onLocalIceCandidate(candidate) } + coroutineScope.launch { + qlcMutext.withLock { + if (candidate != null) { + if (sentSdpOffer) { + listeners.forEach { listener -> listener.onLocalIceCandidate(candidate) } + } else { + queuedLocalCandidates.add(candidate) + } + } + } } } diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt index b17fac04..f00c06d8 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt @@ -25,7 +25,6 @@ import com.fishjamcloud.client.events.UpdateTrackMetadata import com.fishjamcloud.client.events.VadNotification import com.fishjamcloud.client.events.gson import com.fishjamcloud.client.events.serializeToMap -import com.fishjamcloud.client.models.EndpointType import com.fishjamcloud.client.models.Metadata import com.fishjamcloud.client.models.SerializedMediaEvent import com.fishjamcloud.client.models.TrackEncoding @@ -77,12 +76,16 @@ internal class RTCEngineCommunication { fun localCandidate( sdp: String, - sdpMLineIndex: Int + sdpMLineIndex: Int, + sdpMid: Int?, + usernameFragment: String? ) { sendEvent( LocalCandidate( sdp, - sdpMLineIndex + sdpMLineIndex, + sdpMid, + usernameFragment ) ) } @@ -146,7 +149,6 @@ internal class RTCEngineCommunication { listeners.forEach { listener -> listener.onEndpointAdded( event.data.id, - EndpointType.fromString(event.data.type), event.data.metadata ) } diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt index fed80ef2..d6a6c494 100644 --- a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt +++ b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt @@ -3,7 +3,6 @@ package com.fishjamcloud.client.webrtc import com.fishjamcloud.client.events.Endpoint import com.fishjamcloud.client.events.OfferData import com.fishjamcloud.client.events.TrackData -import com.fishjamcloud.client.models.EndpointType import com.fishjamcloud.client.models.Metadata import com.fishjamcloud.client.models.SerializedMediaEvent @@ -17,7 +16,6 @@ internal interface RTCEngineListener { fun onEndpointAdded( endpointId: String, - type: EndpointType, metadata: Metadata? ) @@ -42,7 +40,7 @@ internal interface RTCEngineListener { fun onRemoteCandidate( candidate: String, sdpMLineIndex: Int, - sdpMid: String? + sdpMid: Int? ) fun onTracksAdded( diff --git a/packages/ios-client/Sources/FishjamClient/FishjamClientInternal.swift b/packages/ios-client/Sources/FishjamClient/FishjamClientInternal.swift index ba75fb34..aa64f28c 100644 --- a/packages/ios-client/Sources/FishjamClient/FishjamClientInternal.swift +++ b/packages/ios-client/Sources/FishjamClient/FishjamClientInternal.swift @@ -21,7 +21,7 @@ class FishjamClientInternal { private var _loggerPrefix = "FishjamClientInternal" - private(set) var localEndpoint: Endpoint = Endpoint(id: "", type: .WEBRTC) + private(set) var localEndpoint: Endpoint = Endpoint(id: "") private var prevTracks: [Track] = [] private var remoteEndpointsMap: [String: Endpoint] = [:] @@ -99,7 +99,7 @@ class FishjamClientInternal { } } peerConnectionManager.close() - localEndpoint = Endpoint(id: "", type: .WEBRTC) + localEndpoint = Endpoint(id: "") remoteEndpointsMap = [:] peerConnectionManager.removeListener(self) rtcEngineCommunication.removeListener(self) @@ -522,7 +522,14 @@ extension FishjamClientInternal: PeerConnectionListener { } func onLocalIceCandidate(candidate: RTCIceCandidate) { - rtcEngineCommunication.localCandidate(sdp: candidate.sdp, sdpMLineIndex: candidate.sdpMLineIndex) + let splitSdp = candidate.sdp.split(separator: " ") + guard let ufragIndex = splitSdp.firstIndex(of: "ufrag") else { + return + } + let ufrag = String(splitSdp[ufragIndex + 1]) + rtcEngineCommunication.localCandidate( + sdp: candidate.sdp, sdpMLineIndex: candidate.sdpMLineIndex, sdpMid: Int32(candidate.sdpMid ?? "0") ?? 0, + usernameFragment: ufrag) } } @@ -549,7 +556,7 @@ extension FishjamClientInternal: RTCEngineListener { localEndpoint = localEndpoint.copyWith(id: endpointId) for eventEndpoint in otherEndpoints { var endpoint = Endpoint( - id: eventEndpoint.id, type: EndpointType(fromString: eventEndpoint.type), + id: eventEndpoint.id, metadata: eventEndpoint.metadata ?? Metadata()) for (trackId, track) in eventEndpoint.tracks { let track = Track( @@ -579,11 +586,11 @@ extension FishjamClientInternal: RTCEngineListener { } } - func onEndpointAdded(endpointId: String, type: EndpointType, metadata: Metadata?) { + func onEndpointAdded(endpointId: String, metadata: Metadata?) { if endpointId == localEndpoint.id { return } - let endpoint = Endpoint(id: endpointId, type: type, metadata: metadata ?? Metadata()) + let endpoint = Endpoint(id: endpointId, metadata: metadata ?? Metadata()) remoteEndpointsMap[endpoint.id] = endpoint diff --git a/packages/ios-client/Sources/FishjamClient/events/Event.swift b/packages/ios-client/Sources/FishjamClient/events/Event.swift index d680ca3e..74931a59 100644 --- a/packages/ios-client/Sources/FishjamClient/events/Event.swift +++ b/packages/ios-client/Sources/FishjamClient/events/Event.swift @@ -225,6 +225,8 @@ struct SdpOfferEvent: SendableEvent { struct LocalCandidateEvent: SendableEvent { let candidate: String let sdpMLineIndex: Int32 + let sdpMid: Int32? + let usernameFragment: String? func serialize() -> Payload { return .init([ @@ -234,6 +236,8 @@ struct LocalCandidateEvent: SendableEvent { "data": [ "candidate": candidate, "sdpMLineIndex": sdpMLineIndex, + "sdpMid": sdpMid, + "usernameFragment": usernameFragment, ] as [String: Any], ] as [String: Any], ]) @@ -315,7 +319,6 @@ struct ConnectedEvent: ReceivableEvent, Codable { struct EndpointAddedEvent: ReceivableEvent, Codable { struct Data: Codable { let id: String - let type: EndpointType let metadata: Metadata? let tracks: [String: TrackData]? } @@ -407,7 +410,8 @@ struct RemoteCandidateEvent: ReceivableEvent, Codable { struct Data: Codable { let candidate: String let sdpMLineIndex: Int32 - let sdpMid: String? + let sdpMid: Int32? + let usernameFragment: String? } let type: ReceivableEventType diff --git a/packages/ios-client/Sources/FishjamClient/models/Endpoint.swift b/packages/ios-client/Sources/FishjamClient/models/Endpoint.swift index fe4a9619..09aa506f 100644 --- a/packages/ios-client/Sources/FishjamClient/models/Endpoint.swift +++ b/packages/ios-client/Sources/FishjamClient/models/Endpoint.swift @@ -1,42 +1,19 @@ -public enum EndpointType: String, Codable { - case WEBRTC - - init(fromString s: String) { - self = EndpointType(rawValue: s.uppercased()) ?? .WEBRTC - } - - public init(from decoder: Decoder) throws { - let container = try decoder.singleValueContainer() - let rawValue = try container.decode(String.self) - self.init(fromString: rawValue) - } - - public func encode(to encoder: Swift.Encoder) throws { - var container = encoder.singleValueContainer() - try container.encode(self.rawValue) - } -} - public struct Endpoint { - public let id: String - public let type: EndpointType public let metadata: Metadata public var tracks: [String: Track] - public init(id: String, type: EndpointType, metadata: Metadata = Metadata(), tracks: [String: Track] = [:]) { + public init(id: String, metadata: Metadata = Metadata(), tracks: [String: Track] = [:]) { self.id = id - self.type = type self.metadata = metadata self.tracks = tracks } public func copyWith( - id: String? = nil, type: EndpointType? = nil, metadata: Metadata? = nil, tracks: [String: Track]? = nil + id: String? = nil, metadata: Metadata? = nil, tracks: [String: Track]? = nil ) -> Self { return Endpoint( id: id ?? self.id, - type: type ?? self.type, metadata: metadata ?? self.metadata, tracks: tracks ?? self.tracks ) diff --git a/packages/ios-client/Sources/FishjamClient/webrtc/PeerConnectionManager.swift b/packages/ios-client/Sources/FishjamClient/webrtc/PeerConnectionManager.swift index 078983c2..1cdc123c 100644 --- a/packages/ios-client/Sources/FishjamClient/webrtc/PeerConnectionManager.swift +++ b/packages/ios-client/Sources/FishjamClient/webrtc/PeerConnectionManager.swift @@ -21,9 +21,9 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { private var iceServers: [RTCIceServer] = [] private var config: RTCConfiguration? - private var queuedRemoteCandidates: [RTCIceCandidate] = [] private var midToTrackId: [String: String] = [:] + private static let mediaConstraints = RTCMediaConstraints( mandatoryConstraints: nil, optionalConstraints: ["DtlsSrtpKeyAgreement": kRTCMediaConstraintsValueTrue]) @@ -201,21 +201,12 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { peerConnection.enforceSendOnlyDirection() } - private func drainCandidates() { - for candidate in queuedRemoteCandidates { - connection?.add(candidate, completionHandler: { _ in }) - } - } - /// Parses a list of turn servers and sets them up as `iceServers` that can be used for `RTCPeerConnection` ceration. private func setTurnServers(_ turnServers: [OfferDataEvent.TurnServer]) { - config?.iceTransportPolicy = .relay + let isExWebrtc = turnServers.isEmpty let servers: [RTCIceServer] = turnServers.map { server in - let url = [ - "turn", ":", server.serverAddr, ":", String(server.serverPort), "?transport=", - server.transport, - ].joined() + let url = "turn:\(server.serverAddr):\(server.serverPort)?transport=\(server.transport)" return RTCIceServer( urlStrings: [url], @@ -227,7 +218,7 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { iceServers = servers config = RTCConfiguration() config?.iceServers = servers - config?.iceTransportPolicy = .relay + config?.iceTransportPolicy = isExWebrtc ? .all : .relay } public func close() { @@ -243,9 +234,12 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { // Default ICE server when no turn servers are specified private static func defaultIceServer() -> RTCIceServer { - let iceUrl = "stun:stun.l.google.com:19302" - - return RTCIceServer(urlStrings: [iceUrl]) + let iceUrls = [ + "stun:stun.l.google.com:19302", + "stun:stun.l.google.com:5349", + ] + + return RTCIceServer(urlStrings: iceUrls) } /// On each `OfferData` we receive an information about an amount of audio/video @@ -392,6 +386,7 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { localTracks: [Track], onCompletion: @escaping (_ sdp: String?, _ midToTrackId: [String: String]?, _ error: Error?) -> Void ) { + let isExWebrtc = integratedTurnServers.isEmpty setTurnServers(integratedTurnServers) var needsRestart = true @@ -403,8 +398,8 @@ internal class PeerConnectionManager: NSObject, RTCPeerConnectionDelegate { guard let pc = connection else { return } - - if needsRestart { + + if needsRestart && !isExWebrtc { pc.restartIce() } diff --git a/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineCommunication.swift b/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineCommunication.swift index 6d87659c..ae7efa2e 100644 --- a/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineCommunication.swift +++ b/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineCommunication.swift @@ -40,8 +40,10 @@ internal class RTCEngineCommunication { sendEvent(event: RenegotiateTracksEvent()) } - func localCandidate(sdp: String, sdpMLineIndex: Int32) { - sendEvent(event: LocalCandidateEvent(candidate: sdp, sdpMLineIndex: sdpMLineIndex)) + func localCandidate(sdp: String, sdpMLineIndex: Int32, sdpMid: Int32, usernameFragment: String) { + sendEvent( + event: LocalCandidateEvent( + candidate: sdp, sdpMLineIndex: sdpMLineIndex, sdpMid: sdpMid, usernameFragment: usernameFragment)) } func sdpOffer(sdp: String, trackIdToTrackMetadata: [String: Metadata], midToTrackId: [String: String]) { @@ -75,7 +77,7 @@ internal class RTCEngineCommunication { let endpointAdded = event as! EndpointAddedEvent for listener in listeners { listener.onEndpointAdded( - endpointId: endpointAdded.data.id, type: endpointAdded.data.type, + endpointId: endpointAdded.data.id, metadata: endpointAdded.data.metadata) } case .EndpointRemoved: @@ -101,9 +103,11 @@ internal class RTCEngineCommunication { case .Candidate: let candidate = event as! RemoteCandidateEvent for listener in listeners { + let sdpMid = candidate.data.sdpMid.map(String.init) + listener.onRemoteCandidate( candidate: candidate.data.candidate, sdpMLineIndex: candidate.data.sdpMLineIndex, - sdpMid: candidate.data.sdpMid) + sdpMid: sdpMid) } case .TracksAdded: let tracksAdded = event as! TracksAddedEvent diff --git a/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineListener.swift b/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineListener.swift index 4314254e..b4f01549 100644 --- a/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineListener.swift +++ b/packages/ios-client/Sources/FishjamClient/webrtc/RTCEngineListener.swift @@ -1,7 +1,7 @@ internal protocol RTCEngineListener: AnyObject { func onSendMediaEvent(event: SerializedMediaEvent) func onConnected(endpointId: String, otherEndpoints: [EventEndpoint]) - func onEndpointAdded(endpointId: String, type: EndpointType, metadata: Metadata?) + func onEndpointAdded(endpointId: String, metadata: Metadata?) func onEndpointRemoved(endpointId: String) func onEndpointUpdated(endpointId: String, metadata: Metadata?) func onOfferData(integratedTurnServers: [OfferDataEvent.TurnServer], tracksTypes: [String: Int]) diff --git a/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClient.kt b/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClient.kt index b37ab217..1427aeb8 100644 --- a/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClient.kt +++ b/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClient.kt @@ -426,7 +426,6 @@ class RNFishjamClient( mapOf( "id" to endpoint.id, "isLocal" to (endpoint.id == fishjamClient.getLocalEndpoint().id), - "type" to endpoint.type, "metadata" to endpoint.metadata, "tracks" to endpoint.tracks.values.mapNotNull { track -> diff --git a/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClientModule.kt b/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClientModule.kt index e1d65f0c..cfe1311f 100644 --- a/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClientModule.kt +++ b/packages/react-native-client/android/src/main/java/io/fishjam/reactnative/RNFishjamClientModule.kt @@ -294,7 +294,7 @@ class RNFishjamClientModule : Module() { } AsyncFunction("changeWebRTCLoggingSeverity") Coroutine { severity: String -> - CoroutineScope(Dispatchers.Main).launch { + withContext(Dispatchers.Main) { rnFishjamClient.changeWebRTCLoggingSeverity(severity) } } diff --git a/packages/react-native-client/ios/RNFishjamClient.swift b/packages/react-native-client/ios/RNFishjamClient.swift index 3a43e75e..5820f91a 100644 --- a/packages/react-native-client/ios/RNFishjamClient.swift +++ b/packages/react-native-client/ios/RNFishjamClient.swift @@ -486,7 +486,6 @@ class RNFishjamClient: FishjamClientListener { [ "id": endpoint.id, "isLocal": endpoint.id == RNFishjamClient.fishjamClient!.getLocalEndpoint().id, - "type": endpoint.type, "metadata": endpoint.metadata.toDict(), "tracks": endpoint.tracks.values.compactMap { track -> [String: Any?]? in switch track {