FCE-953: Apply simulcast to protobuf changes (#280)

## Description - Applied and fixed simulcast issues - Small refactor of PeerConnectionManager ## Motivation and Context - Simulcast was not working ## How has this been tested? - Tested with web, android, ios ## Types of changes - [x] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) ## Checklist: - [x] My code follows the code style of this project. - [ ] My change requires a change to the documentation. - [ ] I have updated the documentation accordingly.
fishjam-cloud · Jan 10, 2025 · c88091d · c88091d
1 parent d384abe
commit c88091d
Show file tree

Hide file tree

Showing 29 changed files with 450 additions and 266 deletions.
diff --git a/examples/fishjam-chat/components/VideosGrid/parsePeersToTracks.ts b/examples/fishjam-chat/components/VideosGrid/parsePeersToTracks.ts
@@ -12,7 +12,7 @@ const createGridTracksFromPeer = (
       ...peer.cameraTrack,
       peerId: peer.id,
       isLocal: peer.isLocal,
-      userName: peer.metadata.peer.displayName,
+      userName: peer.metadata.peer?.displayName,
       isVadActive: peer.microphoneTrack?.vadStatus === 'speech',
     });
   }
@@ -22,7 +22,7 @@ const createGridTracksFromPeer = (
       ...peer.screenShareVideoTrack,
       peerId: peer.id,
       isLocal: peer.isLocal,
-      userName: peer.metadata.peer.displayName,
+      userName: peer.metadata.peer?.displayName,
       isVadActive: peer.screenShareAudioTrack?.vadStatus === 'speech',
     });
   }

diff --git a/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx b/examples/fishjam-chat/screens/PreviewScreen/PreviewScreen.tsx
@@ -64,7 +64,7 @@ function PreviewScreen({
 
   useEffect(() => {
     prepareCamera({
-      simulcastEnabled: false,
+      simulcastEnabled: true,
       quality: 'HD169',
       cameraEnabled: true,
     });

diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClient.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClient.kt
@@ -226,7 +226,7 @@ class FishjamClient(
    * Returns current connection stats
    * @return a map containing statistics
    */
-  fun getStats(): Map<String, RTCStats> = client.getStats()
+  suspend fun getStats(): Map<String, RTCStats> = client.getStats()
 
   fun getRemotePeers(): List<Peer> = client.getRemotePeers()
 

diff --git a/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt b/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/FishjamClientInternal.kt
@@ -27,6 +27,7 @@ import com.fishjamcloud.client.webrtc.PeerConnectionListener
 import com.fishjamcloud.client.webrtc.PeerConnectionManager
 import com.fishjamcloud.client.webrtc.RTCEngineCommunication
 import com.fishjamcloud.client.webrtc.RTCEngineListener
+import com.fishjamcloud.client.webrtc.helpers.TrackBitratesMapper
 import fishjam.PeerNotifications
 import fishjam.media_events.server.Server
 import kotlinx.coroutines.CoroutineScope
@@ -225,13 +226,20 @@ internal class FishjamClientInternal(
     endpoints.forEach {
       val (endpointId, endpointData) = it
       if (endpointId == endpointID) {
-        this.localEndpoint = this.localEndpoint.copy(metadata = endpointData.metadataJson.serializeToMap())
+        this.localEndpoint =
+          this.localEndpoint.copy(metadata = endpointData.metadataJson.serializeToMap())
       } else {
         var endpoint = Endpoint(endpointId, endpointData.metadataJson.serializeToMap())
 
         for ((trackId, trackData) in endpointData.trackIdToTrackMap) {
           val track =
-            Track(null, endpointId, trackId, trackData.metadataJson.serializeToMap())
+            Track(
+              mediaTrack = null,
+              sendEncodings = emptyList(),
+              endpointId = endpointId,
+              rtcEngineId = trackId,
+              metadata = trackData.metadataJson.serializeToMap()
+            )
           endpoint = endpoint.addOrReplaceTrack(track)
           this.listener.onTrackAdded(track)
         }
@@ -380,7 +388,14 @@ internal class FishjamClientInternal(
         mediaProjectionPermission
       )
     val screenShareTrack =
-      LocalScreenShareTrack(webrtcTrack, localEndpoint.id, metadata, capturer, videoParameters, videoSource)
+      LocalScreenShareTrack(
+        webrtcTrack,
+        localEndpoint.id,
+        metadata,
+        capturer,
+        videoParameters,
+        videoSource
+      )
     screenShareTrack.start()
     callback.addCallback {
       if (onEnd != null) {
@@ -524,7 +539,7 @@ internal class FishjamClientInternal(
     Logging.enableLogToDebugOutput(severity)
   }
 
-  fun getStats(): Map<String, RTCStats> = peerConnectionManager.getStats()
+  suspend fun getStats(): Map<String, RTCStats> = peerConnectionManager.getStats()
 
   fun getRemotePeers(): List<Peer> = remoteEndpoints.values.toList()
 
@@ -610,11 +625,13 @@ internal class FishjamClientInternal(
           val videoTrack = LocalVideoTrack(webrtcVideoTrack, track)
           localEndpoint = localEndpoint.addOrReplaceTrack(videoTrack)
         }
+
         is LocalAudioTrack -> {
           val webrtcAudioTrack = peerConnectionFactoryWrapper.createAudioTrack(track.audioSource)
           val audioTrack = LocalAudioTrack(webrtcAudioTrack, track)
           localEndpoint = localEndpoint.addOrReplaceTrack(audioTrack)
         }
+
         is LocalScreenShareTrack -> {
           val webrtcTrack = peerConnectionFactoryWrapper.createVideoTrack(track.videoSource)
           val screenShareTrack = LocalScreenShareTrack(webrtcTrack, track)
@@ -638,7 +655,7 @@ internal class FishjamClientInternal(
           offer.description,
           localEndpoint.tracks.map { (_, track) -> track.webrtcId() to track.metadata }.toMap(),
           offer.midToTrackIdMapping,
-          localEndpoint.tracks.map { (_, track) -> track.webrtcId() to 1500000 }.toMap() // TODO(FCE-953): Update with simulcast
+          TrackBitratesMapper.mapTracksToProtoBitrates(localEndpoint.tracks)
         )
         peerConnectionManager.onSentSdpOffer()
       } catch (e: Exception) {
@@ -685,7 +702,14 @@ internal class FishjamClientInternal(
       if (track != null) {
         track.metadata = trackData.metadataJson.serializeToMap()
       } else {
-        track = Track(null, endpointId, trackId, trackData.metadataJson.serializeToMap())
+        track =
+          Track(
+            mediaTrack = null,
+            sendEncodings = emptyList(),
+            endpointId = endpointId,
+            rtcEngineId = trackId,
+            metadata = trackData.metadataJson.serializeToMap()
+          )
         this.listener.onTrackAdded(track)
       }
       updatedTracks[trackId] = track
@@ -822,7 +846,12 @@ internal class FishjamClientInternal(
     coroutineScope.launch {
       val splitSdp = candidate.sdp.split(" ")
       val ufrag = splitSdp[splitSdp.indexOf("ufrag") + 1]
-      rtcEngineCommunication.localCandidate(candidate.sdp, candidate.sdpMLineIndex, candidate.sdpMid.toInt(), ufrag)
+      rtcEngineCommunication.localCandidate(
+        candidate.sdp,
+        candidate.sdpMLineIndex,
+        candidate.sdpMid.toInt(),
+        ufrag
+      )
     }
   }
 }
diff --git a/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/LocalAudioTrack.kt b/...droid-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/LocalAudioTrack.kt
@@ -9,7 +9,7 @@ class LocalAudioTrack(
   endpointId: String,
   metadata: Metadata,
   internal val audioSource: AudioSource
-) : Track(mediaTrack, endpointId, null, metadata),
+) : Track(mediaTrack, emptyList(), endpointId, null, metadata),
   LocalTrack {
   constructor(
     mediaTrack: AudioTrack,

diff --git a/...roid-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/RemoteAudioTrack.kt b/...roid-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/RemoteAudioTrack.kt
@@ -15,7 +15,7 @@ class RemoteAudioTrack(
   rtcEngineId: String?,
   metadata: Metadata,
   id: String = UUID.randomUUID().toString()
-) : Track(audioTrack, endpointId, rtcEngineId, metadata, id) {
+) : Track(audioTrack, emptyList(), endpointId, rtcEngineId, metadata, id) {
   private var onVadNotificationListener: (OnVoiceActivityChangedListener)? = null
 
   var vadStatus = Server.MediaEvent.VadNotification.Status.STATUS_SILENCE

diff --git a/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/Track.kt b/packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/Track.kt
@@ -2,10 +2,12 @@ package com.fishjamcloud.client.media
 
 import com.fishjamcloud.client.models.Metadata
 import org.webrtc.MediaStreamTrack
+import org.webrtc.RtpParameters
 import java.util.UUID
 
 open class Track(
   internal val mediaTrack: MediaStreamTrack?,
+  internal var sendEncodings: List<RtpParameters.Encoding> = emptyList(),
   val endpointId: String,
   private var rtcEngineId: String?,
   metadata: Metadata = mapOf(),

diff --git a/...es/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/VideoTrack.kt b/...es/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/VideoTrack.kt
@@ -12,6 +12,7 @@ open class VideoTrack(
   id: String = UUID.randomUUID().toString()
 ) : Track(
     videoTrack,
+    emptyList(),
     endpointId,
     rtcEngineId,
     metadata,

diff --git a/...lient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt b/...lient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt
@@ -4,9 +4,6 @@ import com.fishjamcloud.client.media.LocalScreenShareTrack
 import com.fishjamcloud.client.media.LocalVideoTrack
 import com.fishjamcloud.client.media.Track
 import com.fishjamcloud.client.models.Constants
-import com.fishjamcloud.client.models.QualityLimitationDurations
-import com.fishjamcloud.client.models.RTCInboundStats
-import com.fishjamcloud.client.models.RTCOutboundStats
 import com.fishjamcloud.client.models.RTCStats
 import com.fishjamcloud.client.models.SimulcastConfig
 import com.fishjamcloud.client.models.TrackBandwidthLimit
@@ -17,6 +14,7 @@ import com.fishjamcloud.client.utils.createOffer
 import com.fishjamcloud.client.utils.getEncodings
 import com.fishjamcloud.client.utils.setLocalDescription
 import com.fishjamcloud.client.utils.setRemoteDescription
+import com.fishjamcloud.client.webrtc.helpers.BitrateLimiter
 import fishjam.media_events.server.Server
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.SupervisorJob
@@ -25,9 +23,7 @@ import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import org.webrtc.*
 import timber.log.Timber
-import java.math.BigInteger
 import java.util.*
-import kotlin.math.pow
 
 internal class PeerConnectionManager(
   private val peerConnectionFactory: PeerConnectionFactoryWrapper
@@ -99,7 +95,8 @@ internal class PeerConnectionManager(
         }
 
       if (videoParameters?.maxBitrate != null) {
-        applyBitrate(sendEncodings, videoParameters.maxBitrate)
+        applyEncodingBitrates(sendEncodings, videoParameters.maxBitrate)
+        track.sendEncodings = sendEncodings
       }
 
       pc.addTransceiver(
@@ -112,47 +109,14 @@ internal class PeerConnectionManager(
     }
   }
 
-  private fun applyBitrate(
+  private fun applyEncodingBitrates(
     encodings: List<RtpParameters.Encoding>,
     maxBitrate: TrackBandwidthLimit
   ) {
-    when (maxBitrate) {
-      is TrackBandwidthLimit.BandwidthLimit -> splitBitrate(encodings, maxBitrate)
-      is TrackBandwidthLimit.SimulcastBandwidthLimit ->
-        encodings.forEach {
-          val encodingLimit = maxBitrate.limit[it.rid]?.limit ?: 0
-          it.maxBitrateBps = if (encodingLimit == 0) null else encodingLimit * 1024
-        }
-    }
-  }
-
-  private fun splitBitrate(
-    encodings: List<RtpParameters.Encoding>,
-    maxBitrate: TrackBandwidthLimit.BandwidthLimit
-  ) {
-    if (encodings.isEmpty()) {
-      Timber.e("splitBitrate: Attempted to limit bandwidth of the track that doesn't have any encodings")
-      return
-    }
-    if (maxBitrate.limit == 0) {
-      encodings.forEach { it.maxBitrateBps = null }
-      return
-    }
-
-    val k0 = encodings.minByOrNull { it.scaleResolutionDownBy ?: 1.0 }
-
-    val bitrateParts =
-      encodings.sumOf {
-        ((k0?.scaleResolutionDownBy ?: 1.0) / (it.scaleResolutionDownBy ?: 1.0)).pow(
-          2
-        )
-      }
-
-    val x = maxBitrate.limit / bitrateParts
+    val calculatedEncodings = BitrateLimiter.calculateBitrates(encodings, maxBitrate)
 
-    encodings.forEach {
-      it.maxBitrateBps =
-        (x * ((k0?.scaleResolutionDownBy ?: 1.0) / (it.scaleResolutionDownBy ?: 1.0)).pow(2) * 1024).toInt()
+    encodings.zip(calculatedEncodings) { original, calculated ->
+      original.maxBitrateBps = calculated.maxBitrateBps
     }
   }
 
@@ -173,7 +137,7 @@ internal class PeerConnectionManager(
         }
       val params = sender.parameters
 
-      applyBitrate(params.getEncodings(), bandwidthLimit)
+      applyEncodingBitrates(params.getEncodings(), bandwidthLimit)
 
       sender.parameters = params
     }
@@ -543,56 +507,14 @@ internal class PeerConnectionManager(
     Timber.d("Renegotiation needed")
   }
 
-  fun getStats(): Map<String, RTCStats> {
-    peerConnection?.getStats { rtcStatsReport -> extractRelevantStats(rtcStatsReport) }
-    return peerConnectionStats.toMap()
-  }
-
-  private fun extractRelevantStats(rp: RTCStatsReport) {
-    rp.statsMap.values.forEach {
-      if (it.type == "outbound-rtp") {
-        val durations = it.members["qualityLimitationDurations"] as? Map<*, *>
-        val qualityLimitation =
-          QualityLimitationDurations(
-            durations?.get("bandwidth") as? Double ?: 0.0,
-            durations?.get("cpu") as? Double ?: 0.0,
-            durations?.get("none") as? Double ?: 0.0,
-            durations?.get("other") as? Double ?: 0.0
-          )
-
-        val tmp =
-          RTCOutboundStats(
-            it.members["kind"] as? String,
-            it.members["rid"] as? String,
-            it.members["bytesSent"] as? BigInteger,
-            it.members["targetBitrate"] as? Double,
-            it.members["packetsSent"] as? Long,
-            it.members["framesEncoded"] as? Long,
-            it.members["framesPerSecond"] as? Double,
-            it.members["frameWidth"] as? Long,
-            it.members["frameHeight"] as? Long,
-            qualityLimitation
-          )
-
-        peerConnectionStats[it.id as String] = tmp
-      } else if (it.type == "inbound-rtp") {
-        val tmp =
-          RTCInboundStats(
-            it.members["kind"] as? String,
-            it.members["jitter"] as? Double,
-            it.members["packetsLost"] as? Int,
-            it.members["packetsReceived"] as? Long,
-            it.members["bytesReceived"] as? BigInteger,
-            it.members["framesReceived"] as? Int,
-            it.members["frameWidth"] as? Long,
-            it.members["frameHeight"] as? Long,
-            it.members["framesPerSecond"] as? Double,
-            it.members["framesDropped"] as? Long
-          )
-
-        peerConnectionStats[it.id as String] = tmp
+  suspend fun getStats(): Map<String, RTCStats> {
+    val pc =
+      peerConnection ?: run {
+        return emptyMap()
       }
-    }
+    val stats = StatsCollector.getStats(pc)
+
+    return stats
   }
 }
 

diff --git a/...ient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt b/...ient/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt
@@ -112,7 +112,7 @@ internal class RTCEngineCommunication {
     sdp: String,
     trackIdToTrackMetadata: Map<String, Metadata?>,
     midToTrackId: Map<String, String>,
-    trackIdToBitrates: Map<String, Int>
+    trackIdToBitrates: Map<String, fishjam.media_events.peer.Peer.MediaEvent.TrackBitrates>
   ) {
     val mediaEvent =
       fishjam.media_events.peer.Peer.MediaEvent
@@ -127,18 +127,7 @@ internal class RTCEngineCommunication {
                 metadata?.let { gson.toJson(it) } ?: ""
               }
             ).putAllTrackIdToBitrates(
-              trackIdToBitrates.mapValues { (trackId, bitrate) ->
-                fishjam.media_events.peer.Peer.MediaEvent.TrackBitrates
-                  .newBuilder()
-                  .setTrackId(trackId)
-                  .addVariantBitrates(
-                    fishjam.media_events.peer.Peer.MediaEvent.VariantBitrate
-                      .newBuilder()
-                      .setVariant(fishjam.media_events.Shared.Variant.VARIANT_UNSPECIFIED) // TODO(FCE-953): Update with simulcast
-                      .setBitrate(bitrate)
-                      .build()
-                  ).build()
-              }
+              trackIdToBitrates
             ).build()
         ).build()