[FCE-555] Add ex-webrtc support (#177)

## Description This PR adds support for ex-webrtc backend. On both platforms: - ~removed `restartIce()` call - it causes issues with connectivity~ => `restartIce` is needed for non exwebrtc. - changes ice transport type from `.relay` to .`all` - conforms to protocol when sending candidates - disables simulcast in example app - it's currently not supported, but maybe it will be supported in the future, thus I'm not changing the api, it's disabled by default anyway NEW changes: - Removed `endpointType` as it was unused and only created confusion about what should be set. - ExWebRTC/WebRTC is now decided based upon if turn servers are passed (same as it's done on Web) On Android: - fixes a (backend) crash when client sends candidate before sending sdp offer by queueing the candidates - on iOS it's not needed as sdp offer is created on the same thread as candidates. In my opinion it's more of a backend issue but it's quicker to workaround it on our side. - fixes a bug when using `changeWebRTCLoggingSeverity` (found out when debugging something) - changes number strategy when serializing to json, so that it sends integers instead of floats On iOS: - removed some dead code with `queuedRemoteCandidates` ## Motivation and Context https://linear.app/swmansion/issue/FCE-555/investigate-work-required-to-support-elixirwebrtc-in-mobile-app ## How has this been tested? - Get a backend with ex-webrtc (built locally from `ex_webrtc_experimental` of fishjam or deployed to fishjam.work) - Open fishjam dashboard - Test only on physical devices, emulators won't work - currently backend has fixed h264 encoding which is unsupported on android emulator - Test different combinations: two mobiles, mobile + web - Test connecting in different order, screencast, disconnecting and reconnecting, adding and removing tracks --------- Co-authored-by: Miłosz Filimowski <[email protected]> Co-authored-by: Milosz Filimowski <[email protected]>
fishjam-cloud · Oct 30, 2024 · 6aad9c5 · 6aad9c5
1 parent 12d9ce0
commit 6aad9c5
Show file tree

Hide file tree

Showing 17 changed files with 135 additions and 115 deletions.
diff --git a/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx b/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx
@@ -64,7 +64,7 @@ function PreviewScreen({
 
   useEffect(() => {
     prepareCamera({
-      simulcastEnabled: true,
+      simulcastEnabled: false,
       quality: 'HD169',
       cameraEnabled: true,
     });

diff --git a/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt b/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt
@@ -13,7 +13,6 @@ import com.fishjamcloud.client.media.Track
 import com.fishjamcloud.client.models.AuthError
 import com.fishjamcloud.client.models.EncodingReason
 import com.fishjamcloud.client.models.Endpoint
-import com.fishjamcloud.client.models.EndpointType
 import com.fishjamcloud.client.models.Metadata
 import com.fishjamcloud.client.models.Peer
 import com.fishjamcloud.client.models.RTCStats
@@ -60,7 +59,7 @@ internal class FishjamClientInternal(
   private val commandsQueue: CommandsQueue = CommandsQueue()
   private var webSocket: WebSocket? = null
 
-  private var localEndpoint: Endpoint = Endpoint(id = "", type = EndpointType.WEBRTC)
+  private var localEndpoint: Endpoint = Endpoint(id = "")
   private var prevTracks = mutableListOf<Track>()
   private var remoteEndpoints: MutableMap<String, Endpoint> = mutableMapOf()
 
@@ -203,7 +202,7 @@ internal class FishjamClientInternal(
     webSocket = null
     remoteEndpoints = mutableMapOf()
     prevTracks = localEndpoint.tracks.values.toMutableList()
-    localEndpoint = Endpoint(id = "", type = EndpointType.WEBRTC)
+    localEndpoint = Endpoint(id = "")
   }
 
   private fun join() {
@@ -224,7 +223,7 @@ internal class FishjamClientInternal(
     localEndpoint = localEndpoint.copy(id = endpointID)
 
     otherEndpoints.forEach {
-      var endpoint = Endpoint(it.id, EndpointType.fromString(it.type), it.metadata)
+      var endpoint = Endpoint(it.id, it.metadata)
 
       for ((trackId, trackData) in it.tracks) {
         val track = Track(null, it.id, trackId, trackData.metadata ?: mapOf())
@@ -258,7 +257,7 @@ internal class FishjamClientInternal(
       rtcEngineCommunication.disconnect()
       localEndpoint.tracks.values.forEach { (it as? LocalTrack)?.stop() }
       peerConnectionManager.close()
-      localEndpoint = Endpoint(id = "", type = EndpointType.WEBRTC)
+      localEndpoint = Endpoint(id = "")
       remoteEndpoints = mutableMapOf()
       peerConnectionManager.removeListener(this@FishjamClientInternal)
       rtcEngineCommunication.removeListener(this@FishjamClientInternal)
@@ -552,14 +551,13 @@ internal class FishjamClientInternal(
 
   override fun onEndpointAdded(
     endpointId: String,
-    type: EndpointType,
     metadata: Metadata?
   ) {
     if (endpointId == this.localEndpoint.id) {
       return
     }
 
-    val endpoint = Endpoint(endpointId, type, metadata)
+    val endpoint = Endpoint(endpointId, metadata)
 
     remoteEndpoints[endpoint.id] = endpoint
 
@@ -639,6 +637,7 @@ internal class FishjamClientInternal(
           localEndpoint.tracks.map { (_, track) -> track.webrtcId() to track.metadata }.toMap(),
           offer.midToTrackIdMapping
         )
+        peerConnectionManager.onSentSdpOffer()
       } catch (e: Exception) {
         Timber.e(e, "Failed to create an sdp offer")
       }
@@ -648,12 +647,12 @@ internal class FishjamClientInternal(
   override fun onRemoteCandidate(
     candidate: String,
     sdpMLineIndex: Int,
-    sdpMid: String?
+    sdpMid: Int?
   ) {
     coroutineScope.launch {
       val iceCandidate =
         IceCandidate(
-          sdpMid ?: "",
+          sdpMid?.toString() ?: "",
           sdpMLineIndex,
           candidate
         )
@@ -816,7 +815,9 @@ internal class FishjamClientInternal(
 
   override fun onLocalIceCandidate(candidate: IceCandidate) {
     coroutineScope.launch {
-      rtcEngineCommunication.localCandidate(candidate.sdp, candidate.sdpMLineIndex)
+      val splitSdp = candidate.sdp.split(" ")
+      val ufrag = splitSdp[splitSdp.indexOf("ufrag") + 1]
+      rtcEngineCommunication.localCandidate(candidate.sdp, candidate.sdpMLineIndex, candidate.sdpMid.toInt(), ufrag)
     }
   }
 }
diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/events/Event.kt
@@ -3,13 +3,14 @@ package com.fishjamcloud.client.events
 import com.fishjamcloud.client.models.Metadata
 import com.fishjamcloud.client.models.Payload
 import com.fishjamcloud.client.models.SimulcastConfig
-import com.google.gson.Gson
+import com.google.gson.GsonBuilder
 import com.google.gson.JsonParseException
+import com.google.gson.ToNumberPolicy
 import com.google.gson.annotations.SerializedName
 import com.google.gson.reflect.TypeToken
 import timber.log.Timber
 
-internal val gson = Gson()
+internal val gson = GsonBuilder().setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create()
 
 // convert a data class to a map
 internal fun <T> T.serializeToMap(): Map<String, Any?> = convert()
@@ -63,15 +64,17 @@ data class LocalCandidate(
   val type: String,
   val data: Payload
 ) : SendableEvent() {
-  constructor(candidate: String, sdpMLineIndex: Int) :
+  constructor(candidate: String, sdpMLineIndex: Int, sdpMid: Int?, usernameFragment: String?) :
     this(
       "custom",
       mapOf(
         "type" to "candidate",
         "data" to
           mapOf(
             "candidate" to candidate,
-            "sdpMLineIndex" to sdpMLineIndex
+            "sdpMLineIndex" to sdpMLineIndex,
+            "sdpMid" to sdpMid,
+            "usernameFragment" to usernameFragment
           )
       )
     )
@@ -375,7 +378,8 @@ data class RemoteCandidate(
   data class Data(
     val candidate: String,
     val sdpMLineIndex: Int,
-    val sdpMid: String?
+    val sdpMid: Int?,
+    val usernameFragment: String?
   )
 }
 

diff --git a/...ges/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt b/...ges/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/Endpoint.kt
@@ -4,7 +4,6 @@ import com.fishjamcloud.client.media.Track
 
 data class Endpoint(
   val id: String,
-  val type: EndpointType,
   val metadata: Metadata? = mapOf(),
   val tracks: Map<String, Track> = mapOf()
 ) {

diff --git a/...android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt b/...android-client/FishjamClient/src/main/java/com/fishjamcloud/client/models/EndpointType.kt
diff --git a/...lient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt b/...lient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt
@@ -57,6 +57,13 @@ internal class PeerConnectionManager(
 
   private var streamIds: List<String> = listOf(UUID.randomUUID().toString())
 
+  // ex-webrtc backend crashes if it's sent a candidate before sdp offer
+  // we can get local candidates from peer connection any time on a different thread
+  // so we queue them here and send them after sending sdp offer
+  private var sentSdpOffer = false
+  private var queuedLocalCandidates = mutableListOf<IceCandidate>()
+  private var qlcMutext = Mutex()
+
   private fun getSendEncodingsFromConfig(simulcastConfig: SimulcastConfig): List<RtpParameters.Encoding> {
     val sendEncodings = Constants.simulcastEncodings()
     simulcastConfig.activeEncodings.forEach {
@@ -263,28 +270,32 @@ internal class PeerConnectionManager(
       return
     }
 
-    this.iceServers =
-      integratedTurnServers.map {
-        val url =
-          listOf(
-            "turn",
-            ":",
-            it.serverAddr,
-            ":",
-            it.serverPort.toString(),
-            "?transport=",
-            it.transport
-          ).joinToString("")
-
-        PeerConnection.IceServer
-          .builder(url)
-          .setUsername(it.username)
-          .setPassword(it.password)
-          .createIceServer()
-      }
+    val isExWebrtc = integratedTurnServers.isEmpty()
+
+    if (isExWebrtc) {
+      val iceServerList = listOf("stun:stun.l.google.com:19302", "stun:stun.l.google.com:5349")
+      this.iceServers = listOf(PeerConnection.IceServer.builder(iceServerList).createIceServer())
+    } else {
+      this.iceServers =
+        integratedTurnServers.map {
+          val url = "turn:${it.serverAddr}:${it.serverPort}?transport=${it.transport}"
+
+          PeerConnection.IceServer
+            .builder(url)
+            .setUsername(it.username)
+            .setPassword(it.password)
+            .createIceServer()
+        }
+    }
 
     val config = PeerConnection.RTCConfiguration(iceServers)
-    config.iceTransportsType = PeerConnection.IceTransportsType.RELAY
+    config.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN
+    config.iceTransportsType =
+      if (isExWebrtc) {
+        PeerConnection.IceTransportsType.ALL
+      } else {
+        PeerConnection.IceTransportsType.RELAY
+      }
     this.config = config
   }
 
@@ -369,20 +380,27 @@ internal class PeerConnectionManager(
     tracksTypes: Map<String, Int>,
     localTracks: List<Track>
   ): SdpOffer {
+    val isExWebrtc = integratedTurnServers.isEmpty()
+
     qrcMutex.withLock {
       this@PeerConnectionManager.queuedRemoteCandidates = mutableListOf()
     }
+    qlcMutext.withLock {
+      sentSdpOffer = false
+      this.queuedLocalCandidates = mutableListOf()
+    }
     prepareIceServers(integratedTurnServers)
 
     var needsRestart = true
     if (peerConnection == null) {
       setupPeerConnection(localTracks)
       needsRestart = false
     }
+
     peerConnectionMutex.withLock {
       val pc = peerConnection!!
 
-      if (needsRestart) {
+      if (needsRestart && !isExWebrtc) {
         pc.restartIce()
       }
 
@@ -475,9 +493,31 @@ internal class PeerConnectionManager(
     Timber.d("Change ice gathering state to $state")
   }
 
+  suspend fun onSentSdpOffer() {
+    qlcMutext.withLock {
+      sentSdpOffer = true
+      queuedLocalCandidates.forEach { candidate ->
+        listeners.forEach { listener ->
+          listener.onLocalIceCandidate(
+            candidate
+          )
+        }
+      }
+      queuedLocalCandidates = mutableListOf()
+    }
+  }
+
   override fun onIceCandidate(candidate: IceCandidate?) {
-    if (candidate != null) {
-      listeners.forEach { listener -> listener.onLocalIceCandidate(candidate) }
+    coroutineScope.launch {
+      qlcMutext.withLock {
+        if (candidate != null) {
+          if (sentSdpOffer) {
+            listeners.forEach { listener -> listener.onLocalIceCandidate(candidate) }
+          } else {
+            queuedLocalCandidates.add(candidate)
+          }
+        }
+      }
     }
   }
 

diff --git a/...ient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt b/...ient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt
@@ -25,7 +25,6 @@ import com.fishjamcloud.client.events.UpdateTrackMetadata
 import com.fishjamcloud.client.events.VadNotification
 import com.fishjamcloud.client.events.gson
 import com.fishjamcloud.client.events.serializeToMap
-import com.fishjamcloud.client.models.EndpointType
 import com.fishjamcloud.client.models.Metadata
 import com.fishjamcloud.client.models.SerializedMediaEvent
 import com.fishjamcloud.client.models.TrackEncoding
@@ -77,12 +76,16 @@ internal class RTCEngineCommunication {
 
   fun localCandidate(
     sdp: String,
-    sdpMLineIndex: Int
+    sdpMLineIndex: Int,
+    sdpMid: Int?,
+    usernameFragment: String?
   ) {
     sendEvent(
       LocalCandidate(
         sdp,
-        sdpMLineIndex
+        sdpMLineIndex,
+        sdpMid,
+        usernameFragment
       )
     )
   }
@@ -146,7 +149,6 @@ internal class RTCEngineCommunication {
         listeners.forEach { listener ->
           listener.onEndpointAdded(
             event.data.id,
-            EndpointType.fromString(event.data.type),
             event.data.metadata
           )
         }

diff --git a/...id-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt b/...id-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineListener.kt
@@ -3,7 +3,6 @@ package com.fishjamcloud.client.webrtc
 import com.fishjamcloud.client.events.Endpoint
 import com.fishjamcloud.client.events.OfferData
 import com.fishjamcloud.client.events.TrackData
-import com.fishjamcloud.client.models.EndpointType
 import com.fishjamcloud.client.models.Metadata
 import com.fishjamcloud.client.models.SerializedMediaEvent
 
@@ -17,7 +16,6 @@ internal interface RTCEngineListener {
 
   fun onEndpointAdded(
     endpointId: String,
-    type: EndpointType,
     metadata: Metadata?
   )
 
@@ -42,7 +40,7 @@ internal interface RTCEngineListener {
   fun onRemoteCandidate(
     candidate: String,
     sdpMLineIndex: Int,
-    sdpMid: String?
+    sdpMid: Int?
   )
 
   fun onTracksAdded(