Merge pull request #91 from android/preload_manager

Add and configure preload manager to exoplayer to improve short form playback experience
android · Sep 6, 2024 · 80ef8a3 · 80ef8a3
2 parents 85d11e8 + 9ac6da7
commit 80ef8a3
Show file tree

Hide file tree

Showing 5 changed files with 337 additions and 26 deletions.
diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.samples.socialite.data.utils
+
+/**
+ *
+ * Sample list of short form video urls, used for preloading multiple videos in background especially used for enabling preload manager of exoplayer.
+ *
+ */
+class ShortsVideoList {
+    companion object {
+        val mediaUris =
+            listOf(
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_4.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_5.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_6.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_7.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_8.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_9.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_10.mp4",
+                "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_11.mp4",
+            )
+    }
+
+    fun get(index: Int): String {
+        return mediaUris[index.mod(mediaUris.size)]
+    }
+}
diff --git a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt
@@ -33,6 +33,7 @@ import com.google.android.samples.socialite.R
 import com.google.android.samples.socialite.data.ChatDao
 import com.google.android.samples.socialite.data.ContactDao
 import com.google.android.samples.socialite.data.MessageDao
+import com.google.android.samples.socialite.data.utils.ShortsVideoList
 import com.google.android.samples.socialite.di.AppCoroutineScope
 import com.google.android.samples.socialite.model.ChatDetail
 import com.google.android.samples.socialite.model.Message
@@ -107,6 +108,12 @@ class ChatRepository @Inject internal constructor(
         )
 
         coroutineScope.launch {
+            // Special incoming message indicating to add shorts videos to try preload in exoplayer
+            if (text == "preload") {
+                preloadShortVideos(chatId, detail, PushReason.IncomingMessage)
+                return@launch
+            }
+
             if (isBotEnabled.firstOrNull() == true) {
                 // Get the previous messages and them generative model chat
                 val pastMessages = getMessageHistory(chatId)
@@ -166,6 +173,26 @@ class ChatRepository @Inject internal constructor(
         }
     }
 
+    /**
+     * Add list of short form videos as sent messages to current chat history.This is used to test the preload manager of exoplayer
+     */
+    private suspend fun preloadShortVideos(
+        chatId: Long,
+        detail: ChatDetail,
+        incomingMessage: PushReason,
+    ) {
+        for ((index, uri) in ShortsVideoList.mediaUris.withIndex())
+            saveMessageAndNotify(
+                chatId,
+                "Shorts $index",
+                0L,
+                uri,
+                "video/mp4",
+                detail,
+                incomingMessage,
+            )
+    }
+
     private suspend fun saveMessageAndNotify(
         chatId: Long,
         text: String,

diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt
@@ -19,7 +19,6 @@ package com.google.android.samples.socialite.ui.home.timeline
 import android.media.MediaMetadataRetriever
 import android.net.Uri
 import android.os.Build
-import androidx.compose.foundation.AndroidExternalSurface
 import androidx.compose.foundation.ExperimentalFoundationApi
 import androidx.compose.foundation.Image
 import androidx.compose.foundation.background
@@ -28,7 +27,6 @@ import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.PaddingValues
 import androidx.compose.foundation.layout.Row
-import androidx.compose.foundation.layout.aspectRatio
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.layout.size
@@ -53,16 +51,20 @@ import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.graphicsLayer
 import androidx.compose.ui.layout.ContentScale
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalInspectionMode
 import androidx.compose.ui.res.painterResource
 import androidx.compose.ui.res.stringResource
 import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.util.lerp
+import androidx.compose.ui.viewinterop.AndroidView
 import androidx.compose.ui.zIndex
 import androidx.hilt.navigation.compose.hiltViewModel
 import androidx.lifecycle.compose.LifecycleResumeEffect
 import androidx.lifecycle.compose.LifecycleStartEffect
 import androidx.media3.common.Player
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.ui.PlayerView
 import coil.compose.AsyncImage
 import coil.request.ImageRequest
 import com.google.android.samples.socialite.R
@@ -71,6 +73,7 @@ import kotlin.math.absoluteValue
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 
+@androidx.annotation.OptIn(UnstableApi::class)
 @Composable
 fun Timeline(
     contentPadding: PaddingValues,
@@ -106,17 +109,17 @@ fun TimelineVerticalPager(
     player: Player?,
     onInitializePlayer: () -> Unit = {},
     onReleasePlayer: () -> Unit = {},
-    onChangePlayerItem: (uri: Uri?) -> Unit = {},
+    onChangePlayerItem: (uri: Uri?, page: Int) -> Unit = { uri: Uri?, i: Int -> },
     videoRatio: Float?,
 ) {
     val pagerState = rememberPagerState(pageCount = { mediaItems.count() })
     LaunchedEffect(pagerState) {
         // Collect from the a snapshotFlow reading the settledPage
         snapshotFlow { pagerState.settledPage }.collect { page ->
             if (mediaItems[page].type == TimelineMediaType.VIDEO) {
-                onChangePlayerItem(Uri.parse(mediaItems[page].uri))
+                onChangePlayerItem(Uri.parse(mediaItems[page].uri), pagerState.currentPage)
             } else {
-                onChangePlayerItem(null)
+                onChangePlayerItem(null, pagerState.currentPage)
             }
         }
     }
@@ -200,23 +203,18 @@ fun TimelinePage(
     when (media.type) {
         TimelineMediaType.VIDEO -> {
             if (page == state.settledPage) {
-                // Use a default 1:1 ratio if the video size is unknown
-                val sanitizedRatio = videoRatio ?: 1f
-                AndroidExternalSurface(
-                    modifier = modifier
-                        .aspectRatio(sanitizedRatio, sanitizedRatio < 1f)
-                        .background(Color.White),
-                ) {
-                    onSurface { surface, _, _ ->
-                        player.setVideoSurface(surface)
-
-                        // Cleanup when surface is destroyed
-                        surface.onDestroyed {
-                            player.clearVideoSurface(this)
-                            release()
-                        }
-                    }
+                // When in preview, early return a Box with the received modifier preserving layout
+                if (LocalInspectionMode.current) {
+                    Box(modifier = modifier)
+                    return
                 }
+                AndroidView(
+                    factory = { PlayerView(it) },
+                    update = { playerView ->
+                        playerView.player = player
+                    },
+                    modifier = modifier.fillMaxSize(),
+                )
             }
         }
         TimelineMediaType.PHOTO -> {

diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt
@@ -18,25 +18,31 @@ package com.google.android.samples.socialite.ui.home.timeline
 
 import android.content.Context
 import android.net.Uri
+import android.os.HandlerThread
+import android.os.Process
+import android.util.Log
 import androidx.annotation.OptIn
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.setValue
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
+import androidx.media3.common.C
 import androidx.media3.common.MediaItem
 import androidx.media3.common.Player
 import androidx.media3.common.VideoSize
 import androidx.media3.common.util.UnstableApi
 import androidx.media3.exoplayer.DefaultLoadControl
 import androidx.media3.exoplayer.ExoPlayer
 import com.google.android.samples.socialite.repository.ChatRepository
+import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadManagerWrapper
 import dagger.hilt.android.lifecycle.HiltViewModel
 import dagger.hilt.android.qualifiers.ApplicationContext
 import javax.inject.Inject
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.launch
 
+@UnstableApi
 @HiltViewModel
 class TimelineViewModel @Inject constructor(
     @ApplicationContext private val application: Context,
@@ -47,11 +53,21 @@ class TimelineViewModel @Inject constructor(
 
     // Single player instance - in the future, we can implement a pool of players to improve
     // latency and allow for concurrent playback
-    var player by mutableStateOf<Player?>(null)
+    var player by mutableStateOf<ExoPlayer?>(null)
 
     // Width/Height ratio of the current media item, used to properly size the Surface
     var videoRatio by mutableStateOf<Float?>(null)
 
+    // Preload Manager for preloaded multiple videos
+    private val enablePreloadManager: Boolean = true
+    private lateinit var preloadManager: PreloadManagerWrapper
+
+    // Playback thread; Internal playback / preload operations are running on the playback thread.
+    private val playerThread: HandlerThread =
+        HandlerThread("playback-thread", Process.THREAD_PRIORITY_AUDIO)
+
+    var playbackStartTimeMs = C.TIME_UNSET
+
     private val videoSizeListener = object : Player.Listener {
         override fun onVideoSizeChanged(videoSize: VideoSize) {
             videoRatio = if (videoSize.height > 0 && videoSize.width > 0) {
@@ -63,6 +79,14 @@ class TimelineViewModel @Inject constructor(
         }
     }
 
+    private val firstFrameListener = object : Player.Listener {
+        override fun onRenderedFirstFrame() {
+            val timeToFirstFrameMs = System.currentTimeMillis() - playbackStartTimeMs
+            Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrameMs ")
+            super.onRenderedFirstFrame()
+        }
+    }
+
     init {
         viewModelScope.launch {
             val allChats = repository.getChats().first()
@@ -96,39 +120,90 @@ class TimelineViewModel @Inject constructor(
 
         // Reduced buffer durations since the primary use-case is for short-form videos
         val loadControl =
-            DefaultLoadControl.Builder().setBufferDurationsMs(500, 1000, 0, 500).build()
+            DefaultLoadControl.Builder().setBufferDurationsMs(5_000, 20_000, 5_00, DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS)
+                .setPrioritizeTimeOverSizeThresholds(true).build()
+
+        playerThread.start()
+
         val newPlayer = ExoPlayer
             .Builder(application.applicationContext)
             .setLoadControl(loadControl)
+            .setPlaybackLooper(playerThread.looper)
             .build()
             .also {
                 it.repeatMode = ExoPlayer.REPEAT_MODE_ONE
                 it.playWhenReady = true
                 it.addListener(videoSizeListener)
+                it.addListener(firstFrameListener)
             }
 
         videoRatio = null
         player = newPlayer
+
+        if (enablePreloadManager) {
+            initPreloadManager(loadControl, playerThread)
+        }
+    }
+
+    private fun initPreloadManager(
+        loadControl: DefaultLoadControl,
+        preloadAndPlaybackThread: HandlerThread,
+    ) {
+        preloadManager =
+            PreloadManagerWrapper.build(
+                preloadAndPlaybackThread.looper,
+                loadControl,
+                application.applicationContext,
+            )
+        preloadManager.setPreloadWindowSize(5)
+
+        // Add videos to preload
+        if (media.isNotEmpty()) {
+            preloadManager.init(media)
+        }
     }
 
     fun releasePlayer() {
+        if (enablePreloadManager) {
+            preloadManager.release()
+        }
         player?.apply {
             removeListener(videoSizeListener)
+            removeListener(firstFrameListener)
             release()
         }
-
+        playerThread.quit()
         videoRatio = null
         player = null
     }
 
-    fun changePlayerItem(uri: Uri?) {
+    fun changePlayerItem(uri: Uri?, currentPlayingIndex: Int) {
         if (player == null) return
 
         player?.apply {
             stop()
             videoRatio = null
             if (uri != null) {
-                setMediaItem(MediaItem.fromUri(uri))
+                // Set the right source to play
+                val mediaItem = MediaItem.fromUri(uri)
+
+                if (enablePreloadManager) {
+                    val mediaSource = preloadManager.getMediaSource(mediaItem)
+                    Log.d("PreloadManager", "Mediasource $mediaSource ")
+
+                    if (mediaSource == null) {
+                        setMediaItem(mediaItem)
+                    } else {
+                        // Use the preloaded media source
+                        setMediaSource(mediaSource)
+                    }
+                    preloadManager.setCurrentPlayingIndex(currentPlayingIndex)
+                } else {
+                    setMediaItem(mediaItem)
+                }
+
+                playbackStartTimeMs = System.currentTimeMillis()
+                Log.d("PreloadManager", "Video Playing $uri ")
                 prepare()
             } else {
                 clearMediaItems()