diff --git a/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt new file mode 100644 index 00000000..8db06b21 --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/data/utils/ShortsVideoList.kt @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.data.utils + +/** + * + * Sample list of short form video urls, used for preloading multiple videos in background especially used for enabling preload manager of exoplayer. + * + */ +class ShortsVideoList { + companion object { + val mediaUris = + listOf( + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_1.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_2.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_3.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_4.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_5.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_6.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_7.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_8.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_9.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_10.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/shorts_android_developers/shorts_11.mp4", + ) + } + + fun get(index: Int): String { + return mediaUris[index.mod(mediaUris.size)] + } +} diff --git a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt index cf8c3610..65e46ad0 100644 --- a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt +++ b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt @@ -33,6 +33,7 @@ import com.google.android.samples.socialite.R import com.google.android.samples.socialite.data.ChatDao import com.google.android.samples.socialite.data.ContactDao import com.google.android.samples.socialite.data.MessageDao +import com.google.android.samples.socialite.data.utils.ShortsVideoList import com.google.android.samples.socialite.di.AppCoroutineScope import com.google.android.samples.socialite.model.ChatDetail import com.google.android.samples.socialite.model.Message @@ -107,6 +108,12 @@ class ChatRepository @Inject internal constructor( ) coroutineScope.launch { + // Special incoming message indicating to add shorts videos to try preload in exoplayer + if (text == "preload") { + preloadShortVideos(chatId, detail, PushReason.IncomingMessage) + return@launch + } + if (isBotEnabled.firstOrNull() == true) { // Get the previous messages and them generative model chat val pastMessages = getMessageHistory(chatId) @@ -166,6 +173,26 @@ class ChatRepository @Inject internal constructor( } } + /** + * Add list of short form videos as sent messages to current chat history.This is used to test the preload manager of exoplayer + */ + private suspend fun preloadShortVideos( + chatId: Long, + detail: ChatDetail, + incomingMessage: PushReason, + ) { + for ((index, uri) in ShortsVideoList.mediaUris.withIndex()) + saveMessageAndNotify( + chatId, + "Shorts $index", + 0L, + uri, + "video/mp4", + detail, + incomingMessage, + ) + } + private suspend fun saveMessageAndNotify( chatId: Long, text: String, diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt index 87446b22..bed4fe41 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/Timeline.kt @@ -19,7 +19,6 @@ package com.google.android.samples.socialite.ui.home.timeline import android.media.MediaMetadataRetriever import android.net.Uri import android.os.Build -import androidx.compose.foundation.AndroidExternalSurface import androidx.compose.foundation.ExperimentalFoundationApi import androidx.compose.foundation.Image import androidx.compose.foundation.background @@ -28,7 +27,6 @@ import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.Row -import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size @@ -53,16 +51,20 @@ import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp import androidx.compose.ui.util.lerp +import androidx.compose.ui.viewinterop.AndroidView import androidx.compose.ui.zIndex import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.LifecycleResumeEffect import androidx.lifecycle.compose.LifecycleStartEffect import androidx.media3.common.Player +import androidx.media3.common.util.UnstableApi +import androidx.media3.ui.PlayerView import coil.compose.AsyncImage import coil.request.ImageRequest import com.google.android.samples.socialite.R @@ -71,6 +73,7 @@ import kotlin.math.absoluteValue import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +@androidx.annotation.OptIn(UnstableApi::class) @Composable fun Timeline( contentPadding: PaddingValues, @@ -106,7 +109,7 @@ fun TimelineVerticalPager( player: Player?, onInitializePlayer: () -> Unit = {}, onReleasePlayer: () -> Unit = {}, - onChangePlayerItem: (uri: Uri?) -> Unit = {}, + onChangePlayerItem: (uri: Uri?, page: Int) -> Unit = { uri: Uri?, i: Int -> }, videoRatio: Float?, ) { val pagerState = rememberPagerState(pageCount = { mediaItems.count() }) @@ -114,9 +117,9 @@ fun TimelineVerticalPager( // Collect from the a snapshotFlow reading the settledPage snapshotFlow { pagerState.settledPage }.collect { page -> if (mediaItems[page].type == TimelineMediaType.VIDEO) { - onChangePlayerItem(Uri.parse(mediaItems[page].uri)) + onChangePlayerItem(Uri.parse(mediaItems[page].uri), pagerState.currentPage) } else { - onChangePlayerItem(null) + onChangePlayerItem(null, pagerState.currentPage) } } } @@ -200,23 +203,18 @@ fun TimelinePage( when (media.type) { TimelineMediaType.VIDEO -> { if (page == state.settledPage) { - // Use a default 1:1 ratio if the video size is unknown - val sanitizedRatio = videoRatio ?: 1f - AndroidExternalSurface( - modifier = modifier - .aspectRatio(sanitizedRatio, sanitizedRatio < 1f) - .background(Color.White), - ) { - onSurface { surface, _, _ -> - player.setVideoSurface(surface) - - // Cleanup when surface is destroyed - surface.onDestroyed { - player.clearVideoSurface(this) - release() - } - } + // When in preview, early return a Box with the received modifier preserving layout + if (LocalInspectionMode.current) { + Box(modifier = modifier) + return } + AndroidView( + factory = { PlayerView(it) }, + update = { playerView -> + playerView.player = player + }, + modifier = modifier.fillMaxSize(), + ) } } TimelineMediaType.PHOTO -> { diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt index caae0e19..dc8c881d 100644 --- a/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt +++ b/app/src/main/java/com/google/android/samples/socialite/ui/home/timeline/TimelineViewModel.kt @@ -18,12 +18,16 @@ package com.google.android.samples.socialite.ui.home.timeline import android.content.Context import android.net.Uri +import android.os.HandlerThread +import android.os.Process +import android.util.Log import androidx.annotation.OptIn import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import androidx.media3.common.C import androidx.media3.common.MediaItem import androidx.media3.common.Player import androidx.media3.common.VideoSize @@ -31,12 +35,14 @@ import androidx.media3.common.util.UnstableApi import androidx.media3.exoplayer.DefaultLoadControl import androidx.media3.exoplayer.ExoPlayer import com.google.android.samples.socialite.repository.ChatRepository +import com.google.android.samples.socialite.ui.player.preloadmanager.PreloadManagerWrapper import dagger.hilt.android.lifecycle.HiltViewModel import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +@UnstableApi @HiltViewModel class TimelineViewModel @Inject constructor( @ApplicationContext private val application: Context, @@ -47,11 +53,21 @@ class TimelineViewModel @Inject constructor( // Single player instance - in the future, we can implement a pool of players to improve // latency and allow for concurrent playback - var player by mutableStateOf(null) + var player by mutableStateOf(null) // Width/Height ratio of the current media item, used to properly size the Surface var videoRatio by mutableStateOf(null) + // Preload Manager for preloaded multiple videos + private val enablePreloadManager: Boolean = true + private lateinit var preloadManager: PreloadManagerWrapper + + // Playback thread; Internal playback / preload operations are running on the playback thread. + private val playerThread: HandlerThread = + HandlerThread("playback-thread", Process.THREAD_PRIORITY_AUDIO) + + var playbackStartTimeMs = C.TIME_UNSET + private val videoSizeListener = object : Player.Listener { override fun onVideoSizeChanged(videoSize: VideoSize) { videoRatio = if (videoSize.height > 0 && videoSize.width > 0) { @@ -63,6 +79,14 @@ class TimelineViewModel @Inject constructor( } } + private val firstFrameListener = object : Player.Listener { + override fun onRenderedFirstFrame() { + val timeToFirstFrameMs = System.currentTimeMillis() - playbackStartTimeMs + Log.d("PreloadManager", "\t\tTime to first Frame = $timeToFirstFrameMs ") + super.onRenderedFirstFrame() + } + } + init { viewModelScope.launch { val allChats = repository.getChats().first() @@ -96,39 +120,90 @@ class TimelineViewModel @Inject constructor( // Reduced buffer durations since the primary use-case is for short-form videos val loadControl = - DefaultLoadControl.Builder().setBufferDurationsMs(500, 1000, 0, 500).build() + DefaultLoadControl.Builder().setBufferDurationsMs(5_000, 20_000, 5_00, DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS) + .setPrioritizeTimeOverSizeThresholds(true).build() + + playerThread.start() + val newPlayer = ExoPlayer .Builder(application.applicationContext) .setLoadControl(loadControl) + .setPlaybackLooper(playerThread.looper) .build() .also { it.repeatMode = ExoPlayer.REPEAT_MODE_ONE it.playWhenReady = true it.addListener(videoSizeListener) + it.addListener(firstFrameListener) } videoRatio = null player = newPlayer + + if (enablePreloadManager) { + initPreloadManager(loadControl, playerThread) + } + } + + private fun initPreloadManager( + loadControl: DefaultLoadControl, + preloadAndPlaybackThread: HandlerThread, + ) { + preloadManager = + PreloadManagerWrapper.build( + preloadAndPlaybackThread.looper, + loadControl, + application.applicationContext, + ) + preloadManager.setPreloadWindowSize(5) + + // Add videos to preload + if (media.isNotEmpty()) { + preloadManager.init(media) + } } fun releasePlayer() { + if (enablePreloadManager) { + preloadManager.release() + } player?.apply { removeListener(videoSizeListener) + removeListener(firstFrameListener) release() } - + playerThread.quit() videoRatio = null player = null } - fun changePlayerItem(uri: Uri?) { + fun changePlayerItem(uri: Uri?, currentPlayingIndex: Int) { if (player == null) return player?.apply { stop() videoRatio = null if (uri != null) { - setMediaItem(MediaItem.fromUri(uri)) + // Set the right source to play + val mediaItem = MediaItem.fromUri(uri) + + if (enablePreloadManager) { + val mediaSource = preloadManager.getMediaSource(mediaItem) + Log.d("PreloadManager", "Mediasource $mediaSource ") + + if (mediaSource == null) { + setMediaItem(mediaItem) + } else { + // Use the preloaded media source + setMediaSource(mediaSource) + } + preloadManager.setCurrentPlayingIndex(currentPlayingIndex) + } else { + setMediaItem(mediaItem) + } + + playbackStartTimeMs = System.currentTimeMillis() + Log.d("PreloadManager", "Video Playing $uri ") prepare() } else { clearMediaItems() diff --git a/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt new file mode 100644 index 00000000..4cd9f104 --- /dev/null +++ b/app/src/main/java/com/google/android/samples/socialite/ui/player/preloadmanager/PreloadManagerWrapper.kt @@ -0,0 +1,166 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.samples.socialite.ui.player.preloadmanager + +import android.content.Context +import android.os.Looper +import androidx.annotation.MainThread +import androidx.media3.common.C +import androidx.media3.common.MediaItem +import androidx.media3.exoplayer.DefaultLoadControl +import androidx.media3.exoplayer.DefaultRendererCapabilitiesList +import androidx.media3.exoplayer.DefaultRenderersFactory +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager +import androidx.media3.exoplayer.source.preload.DefaultPreloadManager.Status.STAGE_LOADED_TO_POSITION_MS +import androidx.media3.exoplayer.source.preload.TargetPreloadStatusControl +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector +import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter +import com.google.android.samples.socialite.ui.home.timeline.TimelineMediaItem + +/** + * Wrapper class to manage all functionalities of preload manager of exoplayer especially for short form content + */ +@androidx.media3.common.util.UnstableApi +class PreloadManagerWrapper +private constructor( + private val defaultPreloadManager: DefaultPreloadManager, +) { + // Queue of media items to be preloaded. Can be ranked based on ranking data + private val preloadWindow: ArrayDeque> = ArrayDeque() + + // Default window size for preload manager. This defines how many maximum items will be preloaded at a time. + private var preloadWindowMaxSize = 5 + + private var currentPlayingIndex = C.INDEX_UNSET + + // List of all items in our current list of media items to be rendered on the UI + private var mediaItemsList = listOf() + + // Defines when to start preloading next items w.r.t current playing item + private val itemsRemainingToStartNextPreloading = 2 + + /** Builds a preload manager instance with default parameters. Preload manager should use the same looper and load control as the player */ + companion object { + fun build( + playbackLooper: Looper, + loadControl: DefaultLoadControl, + context: Context, + ): PreloadManagerWrapper { + val trackSelector = DefaultTrackSelector(context) + trackSelector.init({}, DefaultBandwidthMeter.getSingletonInstance(context)) + val renderersFactory = DefaultRenderersFactory(context) + val preloadManager = DefaultPreloadManager( + PreloadStatusControl(), + DefaultMediaSourceFactory(context), + trackSelector, + DefaultBandwidthMeter.getSingletonInstance(context), + DefaultRendererCapabilitiesList.Factory(renderersFactory), + loadControl.allocator, + playbackLooper, + ) + return PreloadManagerWrapper(preloadManager) + } + } + + /** Add initial list of videos to the preload manager. */ + fun init(mediaList: List) { + if (mediaList.isEmpty()) { + return + } + setCurrentPlayingIndex(0) + setMediaList(mediaList) + preloadNextItems() + } + + /** Sets the index of the current playing media. */ + fun setCurrentPlayingIndex(currentPlayingItemIndex: Int) { + currentPlayingIndex = currentPlayingItemIndex + defaultPreloadManager.setCurrentPlayingIndex(currentPlayingIndex) + preloadNextItems() + } + + /** Sets the list of media items to be played. Can be set as and when new data is loaded. */ + private fun setMediaList(mediaList: List) { + mediaItemsList = mediaList + } + + /** Add the next set of items to preload, w.r.t to the current playing index. */ + private fun preloadNextItems() { + var lastPreloadedIndex = 0 + if (!preloadWindow.isEmpty()) { + lastPreloadedIndex = preloadWindow.last().second + } + + if (lastPreloadedIndex - currentPlayingIndex <= itemsRemainingToStartNextPreloading) { + for (i in 1 until (preloadWindowMaxSize - itemsRemainingToStartNextPreloading)) { + addMediaItem(index = lastPreloadedIndex + i) + removeMediaItem() + } + } + // With invalidate, preload manager will internally sort the priorities of all the media items added to it, and trigger the preload from the most important one. + defaultPreloadManager.invalidate() + } + + /** Remove media item from the preload window. */ + private fun removeMediaItem() { + if (preloadWindow.size <= preloadWindowMaxSize) { + return + } + val itemAndIndex = preloadWindow.removeFirst() + defaultPreloadManager.remove(itemAndIndex.first) + } + + /** Add media item from the preload window. */ + private fun addMediaItem(index: Int) { + if (index < 0 || index >= mediaItemsList.size) { + return + } + + val mediaItem = (MediaItem.fromUri(mediaItemsList[index].uri)) + defaultPreloadManager.add(mediaItem, index) + preloadWindow.addLast(Pair(mediaItem, index)) + } + + /** Sets the size of the Preload Queue. */ + fun setPreloadWindowSize(size: Int) { + preloadWindowMaxSize = size + } + + /** Releases the preload manager. This must be called on the main thread */ + @MainThread + fun release() { + defaultPreloadManager.release() + preloadWindow.clear() + mediaItemsList.toMutableList().clear() + } + + /** Retrieve the preloaded media source */ + fun getMediaSource(mediaItem: MediaItem): MediaSource? { + return defaultPreloadManager.getMediaSource(mediaItem) + } + + /** Customize time to preload, by default as per ranking data */ + @androidx.media3.common.util.UnstableApi + class PreloadStatusControl : TargetPreloadStatusControl { + override fun getTargetPreloadStatus(rankingData: Int): DefaultPreloadManager.Status { + // By default preload first 3 seconds of the video + return DefaultPreloadManager.Status(STAGE_LOADED_TO_POSITION_MS, 3000L) + } + } +}