[FCE-566] Add prop to disable video player when not visible to user (#…

…170)

## Description

This PR adds prop which can disable video player if out of screen.

## Motivation and Context

If the user can't see the video player then there's no need to waste
device's power to receive the track.

## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to
      not work as expected)

## Checklist:

- [x] My code follows the code style of this project.
- [x] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.

---------

Co-authored-by: Miłosz Filimowski <[email protected]>

examples/fishjam-chat/components/VideosGrid/GridTrackItem.tsx

@@ -43,6 +43,7 @@ export const GridTrackItem = ({
       <VideoRendererView
         trackId={track.id}
         videoLayout="FIT"
+        skipRenderOutsideVisibleArea={false}
         style={styles.flexOne}
       />
       {track.isVadActive && (

packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/media/VideoTrack.kt

@@ -18,10 +18,16 @@ open class VideoTrack(
     id
   ) {
   fun addRenderer(renderer: VideoSink) {
-    this.videoTrack.addSink(renderer)
+    videoTrack.addSink(renderer)
   }
 
   fun removeRenderer(renderer: VideoSink) {
-    this.videoTrack.removeSink(renderer)
+    videoTrack.removeSink(renderer)
+  }
+
+  fun shouldReceive(shouldReceive: Boolean) {
+    if (!videoTrack.isDisposed) {
+      videoTrack.setShouldReceive(shouldReceive)
+    }
   }
 }

packages/ios-client/Sources/FishjamClient/media/Tracks/VideoTrack.swift

@@ -31,4 +31,8 @@ open class VideoTrack: Track {
     func removeRenderer(_ renderer: RTCVideoRenderer) {
         videoTrack.remove(renderer)
     }
+
+    func shouldReceive(_ shouldReceive: Bool) {
+        videoTrack.shouldReceive = shouldReceive
+    }
 }

packages/ios-client/Sources/FishjamClient/ui/VideoView.swift

@@ -31,6 +31,19 @@ public class VideoView: UIView {
         }
     }
 
+    /// If set to `nil` the view will always be rendered
+    public var checkVisibilityTimeInterval: TimeInterval? {
+        didSet {
+            removeCheckVisibilityTimer()
+
+            if checkVisibilityTimeInterval == nil {
+                track?.videoTrack.shouldReceive = true
+            } else {
+                setupCheckVisibilityTimer()
+            }
+        }
+    }
+
     /// Dimensions can change dynamically, either when the device changes the orientation
     /// or when the resolution changes adaptively.
     public private(set) var dimensions: Dimensions? {
@@ -48,6 +61,14 @@ public class VideoView: UIView {
     /// usually should be equal to `frame.size`
     private var viewSize: CGSize
 
+    private var checkVisibilityTimer: Timer?
+
+    private var isVisibleOnScreen: Bool {
+        guard !isEffectivelyHidden else { return false }
+        let globalFrame = convert(frame, to: nil)
+        return UIScreen.main.bounds.intersects(globalFrame)
+    }
+
     override init(frame: CGRect) {
         viewSize = frame.size
         super.init(frame: frame)
@@ -88,10 +109,12 @@ public class VideoView: UIView {
         }
     }
 
-    deinit {
+    public override func removeFromSuperview() {
+        removeCheckVisibilityTimer()
         if let rendererView = rendererView {
             track?.removeRenderer(rendererView)
         }
+        super.removeFromSuperview()
     }
 
     /// Delegate listening for the view's changes such as dimensions.
@@ -111,6 +134,20 @@ public class VideoView: UIView {
         }
     }
 
+    private func setupCheckVisibilityTimer() {
+        guard let checkVisibilityTimeInterval else { return }
+        checkVisibilityTimer = Timer.scheduledTimer(withTimeInterval: checkVisibilityTimeInterval, repeats: true) {
+            [weak self] timer in
+            guard let self else { return }
+            self.track?.shouldReceive(self.isVisibleOnScreen)
+        }
+    }
+
+    private func removeCheckVisibilityTimer() {
+        checkVisibilityTimer?.invalidate()
+        checkVisibilityTimer = nil
+    }
+
     // this somehow fixes a bug where the view would get layouted but somehow
     // the frame size would be a `0` at the time therefore breaking the video display
     override public func layoutSubviews() {

packages/ios-client/Sources/FishjamClient/utils/UIView+Hidden.swift

@@ -0,0 +1,9 @@
+extension UIView {
+    var isEffectivelyHidden: Bool {
+        if isHidden {
+            return true
+        } else {
+            return superview?.isEffectivelyHidden ?? false
+        }
+    }
+}

packages/react-native-client/android/src/main/java/io/fishjam/reactnative/VideoRendererViewModule.kt

@@ -15,13 +15,17 @@ class VideoRendererViewModule : Module() {
           RNFishjamClient.localTracksSwitchListenerManager.remove(view)
         }
 
-        Prop("trackId") { view: VideoRendererView, trackId: String ->
+        Prop("trackId") { view, trackId: String ->
           view.init(trackId)
         }
 
-        Prop("videoLayout") { view: VideoRendererView, videoLayout: String ->
+        Prop("videoLayout") { view, videoLayout: String ->
           view.setVideoLayout(videoLayout)
         }
+
+        Prop("skipRenderOutsideVisibleArea") { view, skipRenderOutsideVisibleArea: Boolean ->
+          view.checkVisibilityDelayMillis = if (skipRenderOutsideVisibleArea) 1000 else null
+        }
       }
     }
 }

packages/react-native-client/android/src/main/java/io/fishjam/reactnative/VideoView.kt

@@ -2,16 +2,18 @@ package io.fishjam.reactnative
 
 import android.animation.ValueAnimator
 import android.content.Context
+import android.content.res.Resources
 import android.graphics.Color
+import android.graphics.Rect
 import android.graphics.drawable.ColorDrawable
+import android.os.Handler
+import android.os.Looper
 import android.view.animation.LinearInterpolator
 import com.fishjamcloud.client.media.LocalVideoTrack
 import com.fishjamcloud.client.media.VideoTrack
 import expo.modules.kotlin.AppContext
 import expo.modules.kotlin.views.ExpoView
 import io.fishjam.reactnative.managers.LocalTrackSwitchListener
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import org.webrtc.RendererCommon
 
@@ -35,7 +37,41 @@ abstract class VideoView(
       }
     }
 
-  val coroutineScope: CoroutineScope = CoroutineScope(Dispatchers.Main)
+  private val checkVisibilityHandler: Handler = Handler(Looper.getMainLooper())
+
+  // If set to `null` the view will always be rendered
+  var checkVisibilityDelayMillis: Long? = null
+    set(value) {
+      field = value
+      if (value == null) {
+        getVideoTrack()?.shouldReceive(true)
+      } else {
+        checkVisibilityHandler.post(checkVisibility)
+      }
+    }
+
+  private val checkVisibility =
+    object : Runnable {
+      override fun run() {
+        checkVisibilityDelayMillis?.let {
+          getVideoTrack()?.shouldReceive(isVisibleOnScreen)
+          checkVisibilityHandler.postDelayed(this, it)
+        }
+      }
+    }
+
+  private val isVisibleOnScreen: Boolean
+    get() {
+      if (!isShown) {
+        return false
+      }
+
+      val globalVisibleRect = Rect()
+      getGlobalVisibleRect(globalVisibleRect)
+      val displayMetrics = Resources.getSystem().displayMetrics
+      val screen = Rect(0, 0, displayMetrics.widthPixels, displayMetrics.heightPixels)
+      return screen.intersect(globalVisibleRect)
+    }
 
   init {
     RNFishjamClient.localTracksSwitchListenerManager.add(this)
@@ -53,6 +89,7 @@ abstract class VideoView(
   }
 
   open fun dispose() {
+    checkVisibilityHandler.removeCallbacksAndMessages(null)
     videoView.release()
   }
 
@@ -74,5 +111,6 @@ abstract class VideoView(
 
   fun setupTrack() {
     videoView.setMirror((getVideoTrack() as? LocalVideoTrack)?.isFrontCamera() ?: false)
+    checkVisibilityHandler.post(checkVisibility)
   }
 }

packages/react-native-client/ios/VideoPreviewView.swift

@@ -2,15 +2,14 @@ import ExpoModulesCore
 import FishjamCloudClient
 
 class VideoPreviewView: ExpoView, LocalCameraTrackChangedListener {
-    var videoView: VideoView? = nil
-    private var localVideoTrack: LocalCameraTrack? = nil
+    var videoView: VideoView!
+    private var localVideoTrack: LocalCameraTrack?
 
     required init(appContext: AppContext? = nil) {
         super.init(appContext: appContext)
         videoView = VideoView()
-        videoView?.autoresizingMask = [.flexibleWidth, .flexibleHeight]
-        videoView?.clipsToBounds = true
-        addSubview(videoView!)
+        videoView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
+        videoView.clipsToBounds = true
     }
 
     private func trySetLocalCameraTrack() {
@@ -29,16 +28,18 @@ class VideoPreviewView: ExpoView, LocalCameraTrackChangedListener {
                     track is LocalCameraTrack
                 })?.value as? LocalCameraTrack
             self.localVideoTrack?.start()
-            self.videoView?.track = self.localVideoTrack
+            self.videoView.track = self.localVideoTrack
         }
     }
 
     override func willMove(toSuperview newSuperview: UIView?) {
         super.willMove(toSuperview: newSuperview)
         if newSuperview == nil {
+            videoView.removeFromSuperview()
             RNFishjamClient.localCameraTracksChangedListenersManager.remove(self)
             localVideoTrack?.stop()
         } else {
+            addSubview(videoView)
             RNFishjamClient.localCameraTracksChangedListenersManager.add(self)
             trySetLocalCameraTrack()
         }
@@ -48,26 +49,25 @@ class VideoPreviewView: ExpoView, LocalCameraTrackChangedListener {
         didSet {
             switch videoLayout {
             case "FIT":
-                self.videoView?.layout = .fit
+                videoView.layout = .fit
             case "FILL":
-                self.videoView?.layout = .fill
+                videoView.layout = .fill
             default:
-                self.videoView?.layout = .fill
+                videoView.layout = .fill
             }
         }
     }
 
     var mirrorVideo: Bool = false {
         didSet {
-            self.videoView?.mirror = mirrorVideo
+            videoView.mirror = mirrorVideo
         }
     }
 
     var captureDeviceId: String? = nil {
         didSet {
-            if let captureDeviceId = captureDeviceId {
-                localVideoTrack?.switchCamera(deviceId: captureDeviceId)
-            }
+            guard let captureDeviceId else { return }
+            localVideoTrack?.switchCamera(deviceId: captureDeviceId)
         }
     }
 

packages/react-native-client/ios/VideoRendererView.swift

@@ -3,42 +3,39 @@ import ExpoModulesCore
 import FishjamCloudClient
 
 class VideoRendererView: ExpoView, TrackUpdateListener {
-    var videoView: VideoView? = nil
-    var cancellableEndpoints: Cancellable? = nil
+    var videoView: VideoView!
+    var cancellableEndpoints: Cancellable?
 
     required init(appContext: AppContext? = nil) {
         super.init(appContext: appContext)
-        RNFishjamClient.tracksUpdateListenersManager.add(self)
         videoView = VideoView()
-        videoView?.autoresizingMask = [.flexibleWidth, .flexibleHeight]
-        videoView?.clipsToBounds = true
-        addSubview(videoView!)
-    }
-
-    deinit {
-        RNFishjamClient.tracksUpdateListenersManager.remove(self)
+        videoView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
+        videoView.clipsToBounds = true
     }
 
     override func willMove(toSuperview newSuperview: UIView?) {
         super.willMove(toSuperview: newSuperview)
 
-        if newSuperview != nil {
+        if newSuperview == nil {
+            videoView.removeFromSuperview()
+            RNFishjamClient.tracksUpdateListenersManager.remove(self)
+        } else {
+            addSubview(videoView)
+            RNFishjamClient.tracksUpdateListenersManager.add(self)
             updateVideoTrack()
         }
     }
 
     func updateVideoTrack() {
-        DispatchQueue.main.async {
-            if self.superview != nil {
-                for endpoint in RNFishjamClient.getLocalAndRemoteEndpoints() {
-                    if let track = endpoint.tracks[self.trackId] as? VideoTrack {
-                        if let track = track as? LocalCameraTrack {
-                            self.mirrorVideo = track.isFrontCamera
-                        }
-                        self.videoView?.track = track
-                        return
-                    }
+        DispatchQueue.main.async { [weak self] in
+            guard let self, self.superview != nil else { return }
+            for endpoint in RNFishjamClient.getLocalAndRemoteEndpoints() {
+                guard let track = endpoint.tracks[self.trackId] as? VideoTrack else { continue }
+                if let track = track as? LocalCameraTrack {
+                    self.mirrorVideo = track.isFrontCamera
                 }
+                self.videoView.track = track
+                return
             }
         }
     }
@@ -57,19 +54,25 @@ class VideoRendererView: ExpoView, TrackUpdateListener {
         didSet {
             switch videoLayout {
             case "FIT":
-                self.videoView?.layout = .fit
+                videoView.layout = .fit
             case "FILL":
-                self.videoView?.layout = .fill
+                videoView.layout = .fill
             default:
-                self.videoView?.layout = .fill
+                videoView.layout = .fill
             }
 
         }
     }
 
     var mirrorVideo: Bool = false {
         didSet {
-            self.videoView?.mirror = mirrorVideo
+            videoView.mirror = mirrorVideo
+        }
+    }
+
+    var checkVisibilityTimeInterval: TimeInterval? {
+        didSet {
+            videoView.checkVisibilityTimeInterval = checkVisibilityTimeInterval
         }
     }
 }

packages/react-native-client/ios/VideoRendererViewModule.swift

@@ -5,12 +5,16 @@ public class VideoRendererViewModule: Module {
         Name("VideoRendererViewModule")
 
         View(VideoRendererView.self) {
-            Prop("trackId") { (view: VideoRendererView, prop: String) in
-                view.trackId = prop
+            Prop("trackId") { (view, trackId) in
+                view.trackId = trackId
             }
 
-            Prop("videoLayout") { (view: VideoRendererView, prop: String) in
-                view.videoLayout = prop
+            Prop("videoLayout") { (view, videoLayout) in
+                view.videoLayout = videoLayout
+            }
+
+            Prop("skipRenderOutsideVisibleArea") { (view, skipRenderOutsideVisibleArea) in
+                view.checkVisibilityTimeInterval = skipRenderOutsideVisibleArea ? 1 : nil
             }
         }
     }