From d504228a80e950d2e558e495a45137f790b4e4ba Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Mon, 12 Aug 2024 15:12:42 +0530 Subject: [PATCH 1/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. --- .../socialite/data/utils/ShortsVideoList.kt | 45 ++++++ .../socialite/repository/ChatRepository.kt | 27 ++++ .../socialite/ui/home/timeline/Timeline.kt | 40 +++--- .../ui/home/timeline/TimelineViewModel.kt | 50 ++++++- .../preloadmanager/PreloadManagerWrapper.kt | 128 ++++++++++++++++++ 5 files changed, 266 insertions(+), 24 deletions(-) create mode 100644 app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt create mode 100644 app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt new file mode 100644 index 00000000..6630514f --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.data.utils + +/** + * + * Sample list of short form video urls, used for preloading multiple videos in background especially used for enabling preload manager of exoplayer. + * + */ +class ShortsVideoList { + companion object { + val mediaUris = + mutableListOf( + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_4.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_5.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_6.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_7.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_8.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_9.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_10.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_11.mp4", + ) + } + + fun get(index: Int): String { + return mediaUris[index.mod(mediaUris.size)] + } +} diff --git a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt index cf8c3610..65e46ad0 100644 --- a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt +++ b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt @@ -33,6 +33,7 @@ import com.google.android.samples.socialite.R import com.google.android.samples.socialite.data.ChatDao import com.google.android.samples.socialite.data.ContactDao import com.google.android.samples.socialite.data.MessageDao +import com.google.android.samples.socialite.data.utils.ShortsVideoList import com.google.android.samples.socialite.di.AppCoroutineScope import com.google.android.samples.socialite.model.ChatDetail import com.google.android.samples.socialite.model.Message @@ -107,6 +108,12 @@ class ChatRepository @Inject internal constructor( ) coroutineScope.launch { + // Special incoming message indicating to add shorts videos to try preload in exoplayer + if (text == "preload") { + preloadShortVideos(chatId, detail, PushReason.IncomingMessage) + return@launch + } + if (isBotEnabled.firstOrNull() == true) { // Get the previous messages and them generative model chat val pastMessages = getMessageHistory(chatId) @@ -166,6 +173,26 @@ class ChatRepository @Inject internal constructor( } } + /** + * Add list of short form videos as sent messages to current chat history.This is used to test the preload manager of exoplayer + */ + private suspend fun preloadShortVideos( + chatId: Long, + detail: ChatDetail, + incomingMessage: PushReason, + ) { + for ((index, uri) in ShortsVideoList.mediaUris.withIndex()) + saveMessageAndNotify( + chatId, + "Shorts $index", + 0L, + uri, + "video/mp4", + detail, + incomingMessage, + ) + } + private suspend fun saveMessageAndNotify( chatId: Long, text: String, diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt index 87446b22..bed4fe41 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt @@ -19,7 +19,6 @@ package com.google.android.samples.socialite.ui.home.timeline import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Build -import androidx.compose.foundation.AndroidExternalSurface import androidx.compose.foundation.ExperimentalFoundationApi import androidx.compose.foundation.Image import androidx.compose.foundation.background @@ -28,7 +27,6 @@ import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size @@ -53,16 +51,20 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.compose.ui.util.lerp +import androidx.compose.ui.viewinterop.AndroidView import androidx.compose.ui.zIndex import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.LifecycleStartEffect import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.ui.PlayerView import coil.compose.AsyncImage import coil.request.ImageRequest import com.google.android.samples.socialite.R @@ -71,6 +73,7 @@ import kotlin.math.absoluteValue import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +@androidx.annotation.OptIn(UnstableApi::class) @Composable fun Timeline( contentPadding: PaddingValues, @@ -106,7 +109,7 @@ fun TimelineVerticalPager( player: Player?, onInitializePlayer: () -> Unit = {}, onReleasePlayer: () -> Unit = {}, - onChangePlayerItem: (uri: Uri?) -> Unit = {}, + onChangePlayerItem: (uri: Uri?, page: Int) -> Unit = { uri: Uri?, i: Int -> }, videoRatio: Float?, ) { val pagerState = rememberPagerState(pageCount = { mediaItems.count() }) @@ -114,9 +117,9 @@ fun TimelineVerticalPager( // Collect from the a snapshotFlow reading the settledPage snapshotFlow { pagerState.settledPage }.collect { page -> if (mediaItems[page].type == TimelineMediaType.VIDEO) { - onChangePlayerItem(Uri.parse(mediaItems[page].uri)) + onChangePlayerItem(Uri.parse(mediaItems[page].uri), pagerState.currentPage) } else { - onChangePlayerItem(null) + onChangePlayerItem(null, pagerState.currentPage) } } } @@ -200,23 +203,18 @@ fun TimelinePage( when (media.type) { TimelineMediaType.VIDEO -> { if (page == state.settledPage) { - // Use a default 1:1 ratio if the video size is unknown - val sanitizedRatio = videoRatio ?: 1f - AndroidExternalSurface( - modifier = modifier - .aspectRatio(sanitizedRatio, sanitizedRatio < 1f) - .background(Color.White), - ) { - onSurface { surface, _, _ -> - player.setVideoSurface(surface) - - // Cleanup when surface is destroyed - surface.onDestroyed { - player.clearVideoSurface(this) - release() - } - } + // When in preview, early return a Box with the received modifier preserving layout + if (LocalInspectionMode.current) { + Box(modifier = modifier) + return } + AndroidView( + factory = { PlayerView(it) }, + update = { playerView -> + playerView.player = player + }, + modifier = modifier.fillMaxSize(), + ) } } TimelineMediaType.PHOTO -> { diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index caae0e19..a9462f7c 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -18,6 +18,7 @@ package com.google.android.samples.socialite.ui.home.timeline import android.content.Context import android.net.Uri +import android.util.Log import androidx.annotation.OptIn import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -31,12 +32,14 @@ import androidx.media3.common.util.UnstableApi import androidx.media3.exoplayer.DefaultLoadControl import androidx.media3.exoplayer.ExoPlayer import com.google.android.samples.socialite.repository.ChatRepository +import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadManagerWrapper import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +@UnstableApi @HiltViewModel class TimelineViewModel @Inject constructor( @ApplicationContext private val application: Context, @@ -52,6 +55,10 @@ class TimelineViewModel @Inject constructor( // Width/Height ratio of the current media item, used to properly size the Surface var videoRatio by mutableStateOf(null) + private val enablePreloadManager: Boolean = true + private lateinit var preloadManager: PreloadManagerWrapper + var timeToFirstFrame = 0L + private val videoSizeListener = object : Player.Listener { override fun onVideoSizeChanged(videoSize: VideoSize) { videoRatio = if (videoSize.height > 0 && videoSize.width > 0) { @@ -63,6 +70,14 @@ class TimelineViewModel @Inject constructor( } } + private val firstFrameListener = object : Player.Listener { + override fun onRenderedFirstFrame() { + timeToFirstFrame = System.currentTimeMillis() - timeToFirstFrame + Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrame ") + super.onRenderedFirstFrame() + } + } + init { viewModelScope.launch { val allChats = repository.getChats().first() @@ -96,7 +111,8 @@ class TimelineViewModel @Inject constructor( // Reduced buffer durations since the primary use-case is for short-form videos val loadControl = - DefaultLoadControl.Builder().setBufferDurationsMs(500, 1000, 0, 500).build() + DefaultLoadControl.Builder().setBufferDurationsMs(5_000, 20_000, 5_00, DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS) + .setPrioritizeTimeOverSizeThresholds(true).build() val newPlayer = ExoPlayer .Builder(application.applicationContext) .setLoadControl(loadControl) @@ -105,10 +121,30 @@ class TimelineViewModel @Inject constructor( it.repeatMode = ExoPlayer.REPEAT_MODE_ONE it.playWhenReady = true it.addListener(videoSizeListener) + it.addListener(firstFrameListener) } videoRatio = null player = newPlayer + + if (enablePreloadManager) { + initPreloadManager(loadControl) + } + } + + private fun initPreloadManager(loadControl: DefaultLoadControl) { + preloadManager = + PreloadManagerWrapper.build( + (player as ExoPlayer).applicationLooper, + loadControl, + application.applicationContext, + ) + preloadManager.setPreloadWindowSize(5) + + // Add videos to preload + if (media.isNotEmpty()) { + preloadManager.init(media) + } } fun releasePlayer() { @@ -116,12 +152,14 @@ class TimelineViewModel @Inject constructor( removeListener(videoSizeListener) release() } - + if (enablePreloadManager) { + preloadManager.release() + } videoRatio = null player = null } - fun changePlayerItem(uri: Uri?) { + fun changePlayerItem(uri: Uri?, currentPlayingIndex: Int) { if (player == null) return player?.apply { @@ -129,7 +167,13 @@ class TimelineViewModel @Inject constructor( videoRatio = null if (uri != null) { setMediaItem(MediaItem.fromUri(uri)) + timeToFirstFrame = System.currentTimeMillis() + Log.d("PreloadManager", "Video Playing $uri ") prepare() + if (enablePreloadManager) { + preloadManager.setCurrentPlayingIndex(currentPlayingIndex) + preloadManager.addMediaItem(MediaItem.fromUri(uri), currentPlayingIndex) + } } else { clearMediaItems() } diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt new file mode 100644 index 00000000..5eb6c552 --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.ui.player.preloadmanager + +import android.content.Context +import android.os.Looper +import androidx.annotation.MainThread +import androidx.media3.common.MediaItem +import androidx.media3.exoplayer.DefaultLoadControl +import androidx.media3.exoplayer.DefaultRendererCapabilitiesList +import androidx.media3.exoplayer.DefaultRenderersFactory +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager.Status.STAGE_LOADED_TO_POSITION_MS +import androidx.media3.exoplayer.source.preload.TargetPreloadStatusControl +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector +import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter +import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaItem +import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaType + +/** + * Created by Mayuri Khinvasara on 12/08/24. + * Wrapper class to manage all functionalities of preload manager of exoplayer especially for short form content + */ +@androidx.media3.common.util.UnstableApi +class PreloadManagerWrapper +private constructor( + private val defaultPreloadManager: DefaultPreloadManager, +) { + private val preloadWindow: ArrayDeque> = ArrayDeque() + + // Default window size for preload manager + private var preloadWindowSize = 5 + + /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ + companion object { + fun build( + playbackLooper: Looper, + loadControl: DefaultLoadControl, + context: Context, + ): PreloadManagerWrapper { + val trackSelector = DefaultTrackSelector(context) + trackSelector.init({}, DefaultBandwidthMeter.getSingletonInstance(context)) + val renderersFactory = DefaultRenderersFactory(context) + val preloadManager = DefaultPreloadManager( + PreloadStatusControl(), + DefaultMediaSourceFactory(context), + trackSelector, + DefaultBandwidthMeter.getSingletonInstance(context), + DefaultRendererCapabilitiesList.Factory(renderersFactory), + loadControl.allocator, + playbackLooper, + ) + return PreloadManagerWrapper(preloadManager) + } + } + + /** Add initial list of videos to the preload manager. */ + fun init(mediaList: List) { + for ((index, item) in mediaList.withIndex()) { + if (item.type == TimelineMediaType.VIDEO) { + addMediaItem((MediaItem.fromUri(item.uri)), index) + } + } + } + + /** Sets the index of the current playing media. */ + fun setCurrentPlayingIndex(currentPlayingIndex: Int) { + defaultPreloadManager.setCurrentPlayingIndex(currentPlayingIndex) + } + + /** Sets the size of the Preload Queue. */ + fun setPreloadWindowSize(preloadWindowSize1: Int) { + preloadWindowSize = preloadWindowSize1 + } + + /** Add media item to the preload manager */ + fun addMediaItem(mediaItem: MediaItem, index: Int): Boolean { + // item not found in list. This could happen if the list is auto purged or contents refreshed by APIs + if (index < 0) { + return false + } + // item already added , avoid duplicates + if (preloadWindow.contains(Pair(mediaItem, index))) { + return false + } + + // Window full, purge old data added at the start of the queue + if (preloadWindow.size >= preloadWindowSize) { + defaultPreloadManager.remove(preloadWindow.first().first) + preloadWindow.removeFirstOrNull() + } + // Add video to preload list + preloadWindow.add(Pair(mediaItem, index)) + defaultPreloadManager.add(mediaItem, index) + defaultPreloadManager.invalidate() + return true + } + + /** Releases the preload manager. This must be called on the main thread */ + @MainThread + fun release() { + defaultPreloadManager.release() + } + + /** Customize time to preload, by default as per ranking data */ + @androidx.media3.common.util.UnstableApi + class PreloadStatusControl : TargetPreloadStatusControl { + override fun getTargetPreloadStatus(rankingData: Int): DefaultPreloadManager.Status { + // By default preload first 5 seconds of the video + return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 5000L) + } + } +} From 750874d66f85d317aad8103666c8fc3d47a9e60e Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Mon, 12 Aug 2024 15:12:42 +0530 Subject: [PATCH 2/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. (6) Use the preloaded source from PreloadManager (7) Add a flag to enable Preload Manager --- .../socialite/data/utils/ShortsVideoList.kt | 45 ++++++ .../socialite/repository/ChatRepository.kt | 27 ++++ .../socialite/ui/home/timeline/Timeline.kt | 40 +++--- .../ui/home/timeline/TimelineViewModel.kt | 50 ++++++- .../preloadmanager/PreloadManagerWrapper.kt | 128 ++++++++++++++++++ 5 files changed, 266 insertions(+), 24 deletions(-) create mode 100644 app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt create mode 100644 app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt new file mode 100644 index 00000000..6630514f --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.data.utils + +/** + * + * Sample list of short form video urls, used for preloading multiple videos in background especially used for enabling preload manager of exoplayer. + * + */ +class ShortsVideoList { + companion object { + val mediaUris = + mutableListOf( + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_4.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_5.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_6.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_7.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_8.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_9.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_10.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_11.mp4", + ) + } + + fun get(index: Int): String { + return mediaUris[index.mod(mediaUris.size)] + } +} diff --git a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt index cf8c3610..65e46ad0 100644 --- a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt +++ b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt @@ -33,6 +33,7 @@ import com.google.android.samples.socialite.R import com.google.android.samples.socialite.data.ChatDao import com.google.android.samples.socialite.data.ContactDao import com.google.android.samples.socialite.data.MessageDao +import com.google.android.samples.socialite.data.utils.ShortsVideoList import com.google.android.samples.socialite.di.AppCoroutineScope import com.google.android.samples.socialite.model.ChatDetail import com.google.android.samples.socialite.model.Message @@ -107,6 +108,12 @@ class ChatRepository @Inject internal constructor( ) coroutineScope.launch { + // Special incoming message indicating to add shorts videos to try preload in exoplayer + if (text == "preload") { + preloadShortVideos(chatId, detail, PushReason.IncomingMessage) + return@launch + } + if (isBotEnabled.firstOrNull() == true) { // Get the previous messages and them generative model chat val pastMessages = getMessageHistory(chatId) @@ -166,6 +173,26 @@ class ChatRepository @Inject internal constructor( } } + /** + * Add list of short form videos as sent messages to current chat history.This is used to test the preload manager of exoplayer + */ + private suspend fun preloadShortVideos( + chatId: Long, + detail: ChatDetail, + incomingMessage: PushReason, + ) { + for ((index, uri) in ShortsVideoList.mediaUris.withIndex()) + saveMessageAndNotify( + chatId, + "Shorts $index", + 0L, + uri, + "video/mp4", + detail, + incomingMessage, + ) + } + private suspend fun saveMessageAndNotify( chatId: Long, text: String, diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt index 87446b22..bed4fe41 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt @@ -19,7 +19,6 @@ package com.google.android.samples.socialite.ui.home.timeline import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Build -import androidx.compose.foundation.AndroidExternalSurface import androidx.compose.foundation.ExperimentalFoundationApi import androidx.compose.foundation.Image import androidx.compose.foundation.background @@ -28,7 +27,6 @@ import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size @@ -53,16 +51,20 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.compose.ui.util.lerp +import androidx.compose.ui.viewinterop.AndroidView import androidx.compose.ui.zIndex import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.LifecycleStartEffect import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.ui.PlayerView import coil.compose.AsyncImage import coil.request.ImageRequest import com.google.android.samples.socialite.R @@ -71,6 +73,7 @@ import kotlin.math.absoluteValue import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +@androidx.annotation.OptIn(UnstableApi::class) @Composable fun Timeline( contentPadding: PaddingValues, @@ -106,7 +109,7 @@ fun TimelineVerticalPager( player: Player?, onInitializePlayer: () -> Unit = {}, onReleasePlayer: () -> Unit = {}, - onChangePlayerItem: (uri: Uri?) -> Unit = {}, + onChangePlayerItem: (uri: Uri?, page: Int) -> Unit = { uri: Uri?, i: Int -> }, videoRatio: Float?, ) { val pagerState = rememberPagerState(pageCount = { mediaItems.count() }) @@ -114,9 +117,9 @@ fun TimelineVerticalPager( // Collect from the a snapshotFlow reading the settledPage snapshotFlow { pagerState.settledPage }.collect { page -> if (mediaItems[page].type == TimelineMediaType.VIDEO) { - onChangePlayerItem(Uri.parse(mediaItems[page].uri)) + onChangePlayerItem(Uri.parse(mediaItems[page].uri), pagerState.currentPage) } else { - onChangePlayerItem(null) + onChangePlayerItem(null, pagerState.currentPage) } } } @@ -200,23 +203,18 @@ fun TimelinePage( when (media.type) { TimelineMediaType.VIDEO -> { if (page == state.settledPage) { - // Use a default 1:1 ratio if the video size is unknown - val sanitizedRatio = videoRatio ?: 1f - AndroidExternalSurface( - modifier = modifier - .aspectRatio(sanitizedRatio, sanitizedRatio < 1f) - .background(Color.White), - ) { - onSurface { surface, _, _ -> - player.setVideoSurface(surface) - - // Cleanup when surface is destroyed - surface.onDestroyed { - player.clearVideoSurface(this) - release() - } - } + // When in preview, early return a Box with the received modifier preserving layout + if (LocalInspectionMode.current) { + Box(modifier = modifier) + return } + AndroidView( + factory = { PlayerView(it) }, + update = { playerView -> + playerView.player = player + }, + modifier = modifier.fillMaxSize(), + ) } } TimelineMediaType.PHOTO -> { diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index caae0e19..a9462f7c 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -18,6 +18,7 @@ package com.google.android.samples.socialite.ui.home.timeline import android.content.Context import android.net.Uri +import android.util.Log import androidx.annotation.OptIn import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -31,12 +32,14 @@ import androidx.media3.common.util.UnstableApi import androidx.media3.exoplayer.DefaultLoadControl import androidx.media3.exoplayer.ExoPlayer import com.google.android.samples.socialite.repository.ChatRepository +import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadManagerWrapper import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +@UnstableApi @HiltViewModel class TimelineViewModel @Inject constructor( @ApplicationContext private val application: Context, @@ -52,6 +55,10 @@ class TimelineViewModel @Inject constructor( // Width/Height ratio of the current media item, used to properly size the Surface var videoRatio by mutableStateOf(null) + private val enablePreloadManager: Boolean = true + private lateinit var preloadManager: PreloadManagerWrapper + var timeToFirstFrame = 0L + private val videoSizeListener = object : Player.Listener { override fun onVideoSizeChanged(videoSize: VideoSize) { videoRatio = if (videoSize.height > 0 && videoSize.width > 0) { @@ -63,6 +70,14 @@ class TimelineViewModel @Inject constructor( } } + private val firstFrameListener = object : Player.Listener { + override fun onRenderedFirstFrame() { + timeToFirstFrame = System.currentTimeMillis() - timeToFirstFrame + Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrame ") + super.onRenderedFirstFrame() + } + } + init { viewModelScope.launch { val allChats = repository.getChats().first() @@ -96,7 +111,8 @@ class TimelineViewModel @Inject constructor( // Reduced buffer durations since the primary use-case is for short-form videos val loadControl = - DefaultLoadControl.Builder().setBufferDurationsMs(500, 1000, 0, 500).build() + DefaultLoadControl.Builder().setBufferDurationsMs(5_000, 20_000, 5_00, DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS) + .setPrioritizeTimeOverSizeThresholds(true).build() val newPlayer = ExoPlayer .Builder(application.applicationContext) .setLoadControl(loadControl) @@ -105,10 +121,30 @@ class TimelineViewModel @Inject constructor( it.repeatMode = ExoPlayer.REPEAT_MODE_ONE it.playWhenReady = true it.addListener(videoSizeListener) + it.addListener(firstFrameListener) } videoRatio = null player = newPlayer + + if (enablePreloadManager) { + initPreloadManager(loadControl) + } + } + + private fun initPreloadManager(loadControl: DefaultLoadControl) { + preloadManager = + PreloadManagerWrapper.build( + (player as ExoPlayer).applicationLooper, + loadControl, + application.applicationContext, + ) + preloadManager.setPreloadWindowSize(5) + + // Add videos to preload + if (media.isNotEmpty()) { + preloadManager.init(media) + } } fun releasePlayer() { @@ -116,12 +152,14 @@ class TimelineViewModel @Inject constructor( removeListener(videoSizeListener) release() } - + if (enablePreloadManager) { + preloadManager.release() + } videoRatio = null player = null } - fun changePlayerItem(uri: Uri?) { + fun changePlayerItem(uri: Uri?, currentPlayingIndex: Int) { if (player == null) return player?.apply { @@ -129,7 +167,13 @@ class TimelineViewModel @Inject constructor( videoRatio = null if (uri != null) { setMediaItem(MediaItem.fromUri(uri)) + timeToFirstFrame = System.currentTimeMillis() + Log.d("PreloadManager", "Video Playing $uri ") prepare() + if (enablePreloadManager) { + preloadManager.setCurrentPlayingIndex(currentPlayingIndex) + preloadManager.addMediaItem(MediaItem.fromUri(uri), currentPlayingIndex) + } } else { clearMediaItems() } diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt new file mode 100644 index 00000000..5eb6c552 --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.ui.player.preloadmanager + +import android.content.Context +import android.os.Looper +import androidx.annotation.MainThread +import androidx.media3.common.MediaItem +import androidx.media3.exoplayer.DefaultLoadControl +import androidx.media3.exoplayer.DefaultRendererCapabilitiesList +import androidx.media3.exoplayer.DefaultRenderersFactory +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager.Status.STAGE_LOADED_TO_POSITION_MS +import androidx.media3.exoplayer.source.preload.TargetPreloadStatusControl +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector +import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter +import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaItem +import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaType + +/** + * Created by Mayuri Khinvasara on 12/08/24. + * Wrapper class to manage all functionalities of preload manager of exoplayer especially for short form content + */ +@androidx.media3.common.util.UnstableApi +class PreloadManagerWrapper +private constructor( + private val defaultPreloadManager: DefaultPreloadManager, +) { + private val preloadWindow: ArrayDeque> = ArrayDeque() + + // Default window size for preload manager + private var preloadWindowSize = 5 + + /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ + companion object { + fun build( + playbackLooper: Looper, + loadControl: DefaultLoadControl, + context: Context, + ): PreloadManagerWrapper { + val trackSelector = DefaultTrackSelector(context) + trackSelector.init({}, DefaultBandwidthMeter.getSingletonInstance(context)) + val renderersFactory = DefaultRenderersFactory(context) + val preloadManager = DefaultPreloadManager( + PreloadStatusControl(), + DefaultMediaSourceFactory(context), + trackSelector, + DefaultBandwidthMeter.getSingletonInstance(context), + DefaultRendererCapabilitiesList.Factory(renderersFactory), + loadControl.allocator, + playbackLooper, + ) + return PreloadManagerWrapper(preloadManager) + } + } + + /** Add initial list of videos to the preload manager. */ + fun init(mediaList: List) { + for ((index, item) in mediaList.withIndex()) { + if (item.type == TimelineMediaType.VIDEO) { + addMediaItem((MediaItem.fromUri(item.uri)), index) + } + } + } + + /** Sets the index of the current playing media. */ + fun setCurrentPlayingIndex(currentPlayingIndex: Int) { + defaultPreloadManager.setCurrentPlayingIndex(currentPlayingIndex) + } + + /** Sets the size of the Preload Queue. */ + fun setPreloadWindowSize(preloadWindowSize1: Int) { + preloadWindowSize = preloadWindowSize1 + } + + /** Add media item to the preload manager */ + fun addMediaItem(mediaItem: MediaItem, index: Int): Boolean { + // item not found in list. This could happen if the list is auto purged or contents refreshed by APIs + if (index < 0) { + return false + } + // item already added , avoid duplicates + if (preloadWindow.contains(Pair(mediaItem, index))) { + return false + } + + // Window full, purge old data added at the start of the queue + if (preloadWindow.size >= preloadWindowSize) { + defaultPreloadManager.remove(preloadWindow.first().first) + preloadWindow.removeFirstOrNull() + } + // Add video to preload list + preloadWindow.add(Pair(mediaItem, index)) + defaultPreloadManager.add(mediaItem, index) + defaultPreloadManager.invalidate() + return true + } + + /** Releases the preload manager. This must be called on the main thread */ + @MainThread + fun release() { + defaultPreloadManager.release() + } + + /** Customize time to preload, by default as per ranking data */ + @androidx.media3.common.util.UnstableApi + class PreloadStatusControl : TargetPreloadStatusControl { + override fun getTargetPreloadStatus(rankingData: Int): DefaultPreloadManager.Status { + // By default preload first 5 seconds of the video + return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 5000L) + } + } +} From 64686e31feede251b221aa20a68e9bb23a485074 Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Wed, 28 Aug 2024 16:27:05 +0530 Subject: [PATCH 3/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. (6) Use the preloaded source from PreloadManager (7) Add a flag to enable Preload Manager --- .../ui/home/timeline/TimelineViewModel.kt | 27 ++++-- .../preloadmanager/PreloadManagerWrapper.kt | 89 +++++++++++++------ 2 files changed, 83 insertions(+), 33 deletions(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index a9462f7c..0114a70f 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -50,13 +50,15 @@ class TimelineViewModel @Inject constructor( // Single player instance - in the future, we can implement a pool of players to improve // latency and allow for concurrent playback - var player by mutableStateOf(null) + var player by mutableStateOf(null) // Width/Height ratio of the current media item, used to properly size the Surface var videoRatio by mutableStateOf(null) + // Preload Manager for preloaded multiple videos private val enablePreloadManager: Boolean = true private lateinit var preloadManager: PreloadManagerWrapper + var timeToFirstFrame = 0L private val videoSizeListener = object : Player.Listener { @@ -166,14 +168,27 @@ class TimelineViewModel @Inject constructor( stop() videoRatio = null if (uri != null) { - setMediaItem(MediaItem.fromUri(uri)) - timeToFirstFrame = System.currentTimeMillis() - Log.d("PreloadManager", "Video Playing $uri ") - prepare() + // Set the right source to play + val mediaItem = MediaItem.fromUri(uri) + if (enablePreloadManager) { + val mediaSource = preloadManager.getMediaSource(mediaItem) + Log.d("PreloadManager", "Mediasource $mediaSource ") + + if (mediaSource == null) { + setMediaItem(mediaItem) + } else { + // Use the preloaded media source + setMediaSource(mediaSource) + } preloadManager.setCurrentPlayingIndex(currentPlayingIndex) - preloadManager.addMediaItem(MediaItem.fromUri(uri), currentPlayingIndex) + } else { + setMediaItem(mediaItem) } + + timeToFirstFrame = System.currentTimeMillis() + Log.d("PreloadManager", "Video Playing $uri ") + prepare() } else { clearMediaItems() } diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt index 5eb6c552..a25c8ca0 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -19,18 +19,19 @@ package com.google.android.samples.socialite.ui.player.preloadmanager import android.content.Context import android.os.Looper import androidx.annotation.MainThread +import androidx.media3.common.C import androidx.media3.common.MediaItem import androidx.media3.exoplayer.DefaultLoadControl import androidx.media3.exoplayer.DefaultRendererCapabilitiesList import androidx.media3.exoplayer.DefaultRenderersFactory import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.MediaSource import androidx.media3.exoplayer.source.preload.DefaultPreloadManager import androidx.media3.exoplayer.source.preload.DefaultPreloadManager.Status.STAGE_LOADED_TO_POSITION_MS import androidx.media3.exoplayer.source.preload.TargetPreloadStatusControl import androidx.media3.exoplayer.trackselection.DefaultTrackSelector import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaItem -import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaType /** * Created by Mayuri Khinvasara on 12/08/24. @@ -44,7 +45,12 @@ private constructor( private val preloadWindow: ArrayDeque> = ArrayDeque() // Default window size for preload manager - private var preloadWindowSize = 5 + private var preloadWindowMaxSize = 6 + private var currentPlayingIndex = C.INDEX_UNSET + private var mediaItemsList = listOf() + + // Defines when to start preloading next items w.r.t current playing item + private val itemsRemainingToStartNextPreloading = 2 /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ companion object { @@ -71,50 +77,79 @@ private constructor( /** Add initial list of videos to the preload manager. */ fun init(mediaList: List) { - for ((index, item) in mediaList.withIndex()) { - if (item.type == TimelineMediaType.VIDEO) { - addMediaItem((MediaItem.fromUri(item.uri)), index) - } + if (mediaList.isEmpty()) { + return } + setCurrentPlayingIndex(0) + setMediaList(mediaList) + preloadNextItems() } /** Sets the index of the current playing media. */ - fun setCurrentPlayingIndex(currentPlayingIndex: Int) { + fun setCurrentPlayingIndex(currentPlayingItemIndex: Int) { + currentPlayingIndex = currentPlayingItemIndex defaultPreloadManager.setCurrentPlayingIndex(currentPlayingIndex) + preloadNextItems() } - /** Sets the size of the Preload Queue. */ - fun setPreloadWindowSize(preloadWindowSize1: Int) { - preloadWindowSize = preloadWindowSize1 + /** Sets the list of media items to be played. Can be set as and when new data is loaded. */ + private fun setMediaList(mediaList: List) { + mediaItemsList = mediaList } - /** Add media item to the preload manager */ - fun addMediaItem(mediaItem: MediaItem, index: Int): Boolean { - // item not found in list. This could happen if the list is auto purged or contents refreshed by APIs - if (index < 0) { - return false + /** Ensure that current playing item is in the middle of the preload Window . */ + private fun preloadNextItems() { + var lastPreloadedIndex = 0 + if (!preloadWindow.isEmpty()) { + lastPreloadedIndex = preloadWindow.last().second } - // item already added , avoid duplicates - if (preloadWindow.contains(Pair(mediaItem, index))) { - return false + + if (lastPreloadedIndex - currentPlayingIndex <= itemsRemainingToStartNextPreloading) { + for (i in 1 until (preloadWindowMaxSize - itemsRemainingToStartNextPreloading)) { + addMediaItem(index = lastPreloadedIndex + i) + removeMediaItem() + } + // With invalidate, preload manager will internally sort the priorities of all the media items added to it, and trigger the preload from the most important one + defaultPreloadManager.invalidate() } + } - // Window full, purge old data added at the start of the queue - if (preloadWindow.size >= preloadWindowSize) { - defaultPreloadManager.remove(preloadWindow.first().first) - preloadWindow.removeFirstOrNull() + /** Remove media item from the preload window. */ + private fun removeMediaItem() { + if (preloadWindow.size <= preloadWindowMaxSize) { + return } - // Add video to preload list - preloadWindow.add(Pair(mediaItem, index)) + val itemAndIndex = preloadWindow.removeFirst() + defaultPreloadManager.remove(itemAndIndex.first) + } + + /** Add media item from the preload window. */ + private fun addMediaItem(index: Int) { + if (index < 0 || index >= mediaItemsList.size) { + return + } + + val mediaItem = (MediaItem.fromUri(mediaItemsList[index].uri)) defaultPreloadManager.add(mediaItem, index) - defaultPreloadManager.invalidate() - return true + preloadWindow.addLast(Pair(mediaItem, index)) + } + + /** Sets the size of the Preload Queue. */ + fun setPreloadWindowSize(size: Int) { + preloadWindowMaxSize = size } /** Releases the preload manager. This must be called on the main thread */ @MainThread fun release() { defaultPreloadManager.release() + preloadWindow.clear() + mediaItemsList.toMutableList().clear() + } + + /** Retrieve the preloaded media source */ + fun getMediaSource(mediaItem: MediaItem): MediaSource? { + return defaultPreloadManager.getMediaSource(mediaItem) } /** Customize time to preload, by default as per ranking data */ @@ -122,7 +157,7 @@ private constructor( class PreloadStatusControl : TargetPreloadStatusControl { override fun getTargetPreloadStatus(rankingData: Int): DefaultPreloadManager.Status { // By default preload first 5 seconds of the video - return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 5000L) + return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 3000L) } } } From b7204a9b498b2916c7e6eb5526e202e4a8416167 Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Wed, 28 Aug 2024 16:39:44 +0530 Subject: [PATCH 4/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. (6) Use the preloaded source from PreloadManager (7) Add a flag to enable Preload Manager --- .../samples/socialite/data/utils/ShortsVideoList.kt | 2 +- .../samples/socialite/ui/home/timeline/Timeline.kt | 2 +- .../ui/player/preloadmanager/PreloadManagerWrapper.kt | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt index 6630514f..9a5d9d69 100644 --- a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt +++ b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt @@ -24,7 +24,7 @@ package com.google.android.samples.socialite.data.utils class ShortsVideoList { companion object { val mediaUris = - mutableListOf( + listOf( "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4", diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt index bed4fe41..6e616b01 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt @@ -154,7 +154,7 @@ fun TimelineVerticalPager( .fillMaxSize() .padding(8.dp) .clip(RoundedCornerShape(16.dp)) - .background(MaterialTheme.colorScheme.secondaryContainer) + // .background(MaterialTheme.colorScheme.secondaryContainer) .graphicsLayer { // Calculate the absolute offset for the current page from the // scroll position. We use the absolute value which allows us to mirror diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt index a25c8ca0..aa2b526f 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -34,7 +34,6 @@ import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaItem /** - * Created by Mayuri Khinvasara on 12/08/24. * Wrapper class to manage all functionalities of preload manager of exoplayer especially for short form content */ @androidx.media3.common.util.UnstableApi @@ -42,11 +41,15 @@ class PreloadManagerWrapper private constructor( private val defaultPreloadManager: DefaultPreloadManager, ) { + // Queue of media items to be preloaded. Can be ranked based on ranking data private val preloadWindow: ArrayDeque> = ArrayDeque() - // Default window size for preload manager - private var preloadWindowMaxSize = 6 + // Default window size for preload manager. This defines how many maximum items will be preloaded at a time. If more than maximum items are added to the preload window + private var preloadWindowMaxSize = 5 + private var currentPlayingIndex = C.INDEX_UNSET + + // List of all items in our current list of media items to be rendered on the UI private var mediaItemsList = listOf() // Defines when to start preloading next items w.r.t current playing item From b7d3e05a9de10a610b75369de8cf4665b02c3739 Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Thu, 5 Sep 2024 16:00:08 +0530 Subject: [PATCH 5/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. (6) Use the preloaded source from PreloadManager (7) Add a flag to enable Preload Manager --- .../ui/home/timeline/TimelineViewModel.kt | 28 ++++++++++++++----- .../preloadmanager/PreloadManagerWrapper.kt | 12 ++++++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index 0114a70f..cfb263cc 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -18,6 +18,8 @@ package com.google.android.samples.socialite.ui.home.timeline import android.content.Context import android.net.Uri +import android.os.HandlerThread +import android.os.Process import android.util.Log import androidx.annotation.OptIn import androidx.compose.runtime.getValue @@ -25,6 +27,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import androidx.media3.common.C import androidx.media3.common.MediaItem import androidx.media3.common.Player import androidx.media3.common.VideoSize @@ -59,7 +62,11 @@ class TimelineViewModel @Inject constructor( private val enablePreloadManager: Boolean = true private lateinit var preloadManager: PreloadManagerWrapper - var timeToFirstFrame = 0L + // Playback thread; Internal playback / preload operations are running on the playback thread. + private val playerThread: HandlerThread = + HandlerThread("playback-thread", Process.THREAD_PRIORITY_AUDIO) + + var playbackStartTimeMs = C.TIME_UNSET private val videoSizeListener = object : Player.Listener { override fun onVideoSizeChanged(videoSize: VideoSize) { @@ -74,8 +81,8 @@ class TimelineViewModel @Inject constructor( private val firstFrameListener = object : Player.Listener { override fun onRenderedFirstFrame() { - timeToFirstFrame = System.currentTimeMillis() - timeToFirstFrame - Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrame ") + val timeToFirstFrameMs = System.currentTimeMillis() - playbackStartTimeMs + Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrameMs ") super.onRenderedFirstFrame() } } @@ -115,9 +122,13 @@ class TimelineViewModel @Inject constructor( val loadControl = DefaultLoadControl.Builder().setBufferDurationsMs(5_000, 20_000, 5_00, DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS) .setPrioritizeTimeOverSizeThresholds(true).build() + + playerThread.start() + val newPlayer = ExoPlayer .Builder(application.applicationContext) .setLoadControl(loadControl) + .setPlaybackLooper(playerThread.looper) .build() .also { it.repeatMode = ExoPlayer.REPEAT_MODE_ONE @@ -130,14 +141,17 @@ class TimelineViewModel @Inject constructor( player = newPlayer if (enablePreloadManager) { - initPreloadManager(loadControl) + initPreloadManager(loadControl, playerThread) } } - private fun initPreloadManager(loadControl: DefaultLoadControl) { + private fun initPreloadManager( + loadControl: DefaultLoadControl, + preloadAndPlaybackThread: HandlerThread, + ) { preloadManager = PreloadManagerWrapper.build( - (player as ExoPlayer).applicationLooper, + preloadAndPlaybackThread.looper, loadControl, application.applicationContext, ) @@ -186,7 +200,7 @@ class TimelineViewModel @Inject constructor( setMediaItem(mediaItem) } - timeToFirstFrame = System.currentTimeMillis() + playbackStartTimeMs = System.currentTimeMillis() Log.d("PreloadManager", "Video Playing $uri ") prepare() } else { diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt index aa2b526f..f337f5c9 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -44,7 +44,7 @@ private constructor( // Queue of media items to be preloaded. Can be ranked based on ranking data private val preloadWindow: ArrayDeque> = ArrayDeque() - // Default window size for preload manager. This defines how many maximum items will be preloaded at a time. If more than maximum items are added to the preload window + // Default window size for preload manager. This defines how many maximum items will be preloaded at a time. private var preloadWindowMaxSize = 5 private var currentPlayingIndex = C.INDEX_UNSET @@ -57,6 +57,10 @@ private constructor( /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ companion object { + + // Thread on which the preload manager is running + private lateinit var playbackAndPreloadThread: Looper + fun build( playbackLooper: Looper, loadControl: DefaultLoadControl, @@ -65,6 +69,7 @@ private constructor( val trackSelector = DefaultTrackSelector(context) trackSelector.init({}, DefaultBandwidthMeter.getSingletonInstance(context)) val renderersFactory = DefaultRenderersFactory(context) + playbackAndPreloadThread = playbackLooper val preloadManager = DefaultPreloadManager( PreloadStatusControl(), DefaultMediaSourceFactory(context), @@ -100,7 +105,7 @@ private constructor( mediaItemsList = mediaList } - /** Ensure that current playing item is in the middle of the preload Window . */ + /** Add the next set of items to preload, w.r.t to the current playing index. */ private fun preloadNextItems() { var lastPreloadedIndex = 0 if (!preloadWindow.isEmpty()) { @@ -148,6 +153,7 @@ private constructor( defaultPreloadManager.release() preloadWindow.clear() mediaItemsList.toMutableList().clear() + playbackAndPreloadThread.quit() } /** Retrieve the preloaded media source */ @@ -159,7 +165,7 @@ private constructor( @androidx.media3.common.util.UnstableApi class PreloadStatusControl : TargetPreloadStatusControl { override fun getTargetPreloadStatus(rankingData: Int): DefaultPreloadManager.Status { - // By default preload first 5 seconds of the video + // By default preload first 3 seconds of the video return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 3000L) } } From 81f26500c6477faf9ea5c51a6a3295f7ce74d302 Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Thu, 5 Sep 2024 16:07:24 +0530 Subject: [PATCH 6/8] Add preload manager to exoplayer. (1) Customize time frame to preload for videos.(2) Add preload manager as an optional to Timeline screen (4) Add 10 remote videos to chats by sending :prelaod: message to any chat (5) Replace Timeline surface view with Player view for better performance. (6) Use the preloaded source from PreloadManager (7) Add a flag to enable Preload Manager --- .../android/samples/socialite/data/utils/ShortsVideoList.kt | 2 +- .../android/samples/socialite/ui/home/timeline/Timeline.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt index 9a5d9d69..8db06b21 100644 --- a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt +++ b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt @@ -24,7 +24,7 @@ package com.google.android.samples.socialite.data.utils class ShortsVideoList { companion object { val mediaUris = - listOf( + listOf( "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4", diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt index 6e616b01..bed4fe41 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt @@ -154,7 +154,7 @@ fun TimelineVerticalPager( .fillMaxSize() .padding(8.dp) .clip(RoundedCornerShape(16.dp)) - // .background(MaterialTheme.colorScheme.secondaryContainer) + .background(MaterialTheme.colorScheme.secondaryContainer) .graphicsLayer { // Calculate the absolute offset for the current page from the // scroll position. We use the absolute value which allows us to mirror From f90bfab713234465d47e027454d21afc63d0ea5e Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Fri, 6 Sep 2024 12:19:04 +0530 Subject: [PATCH 7/8] Add correct order of releasing resources for player, preload manager and playback thread. --- .../socialite/ui/home/timeline/TimelineViewModel.kt | 9 ++++++--- .../ui/player/preloadmanager/PreloadManagerWrapper.kt | 10 ++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index cfb263cc..d285d2b7 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -39,6 +39,7 @@ import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadMana import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject +import kotlin.math.truncate import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch @@ -164,13 +165,15 @@ class TimelineViewModel @Inject constructor( } fun releasePlayer() { + if (enablePreloadManager) { + preloadManager.release() + } player?.apply { removeListener(videoSizeListener) + removeListener(firstFrameListener) release() } - if (enablePreloadManager) { - preloadManager.release() - } + playerThread.quit() videoRatio = null player = null } diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt index f337f5c9..4cd9f104 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -57,10 +57,6 @@ private constructor( /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ companion object { - - // Thread on which the preload manager is running - private lateinit var playbackAndPreloadThread: Looper - fun build( playbackLooper: Looper, loadControl: DefaultLoadControl, @@ -69,7 +65,6 @@ private constructor( val trackSelector = DefaultTrackSelector(context) trackSelector.init({}, DefaultBandwidthMeter.getSingletonInstance(context)) val renderersFactory = DefaultRenderersFactory(context) - playbackAndPreloadThread = playbackLooper val preloadManager = DefaultPreloadManager( PreloadStatusControl(), DefaultMediaSourceFactory(context), @@ -117,9 +112,9 @@ private constructor( addMediaItem(index = lastPreloadedIndex + i) removeMediaItem() } - // With invalidate, preload manager will internally sort the priorities of all the media items added to it, and trigger the preload from the most important one - defaultPreloadManager.invalidate() } + // With invalidate, preload manager will internally sort the priorities of all the media items added to it, and trigger the preload from the most important one. + defaultPreloadManager.invalidate() } /** Remove media item from the preload window. */ @@ -153,7 +148,6 @@ private constructor( defaultPreloadManager.release() preloadWindow.clear() mediaItemsList.toMutableList().clear() - playbackAndPreloadThread.quit() } /** Retrieve the preloaded media source */ From 9ac6da7d65b9b167342d3f6d5119d59bf9e0f1a4 Mon Sep 17 00:00:00 2001 From: mayurikhin Date: Fri, 6 Sep 2024 12:21:58 +0530 Subject: [PATCH 8/8] Add correct order of releasing resources for player, preload manager and playback thread. --- .../samples/socialite/ui/home/timeline/TimelineViewModel.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index d285d2b7..dc8c881d 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -39,7 +39,6 @@ import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadMana import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject -import kotlin.math.truncate import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch