From 5587309604bcc77f3adc89a131254e07b744cc2a Mon Sep 17 00:00:00 2001 From: sszuev Date: Mon, 20 Jan 2025 18:36:58 +0300 Subject: [PATCH] android: use media3 instead of standard media-player; fix number bugs with playing audio and with stage options --- android/app/build.gradle.kts | 3 +- .../sszuev/flashcards/android/MainActivity.kt | 1 + .../android/models/CardViewModel.kt | 208 +++++++++--------- .../android/models/ViewModelFactory.kt | 10 +- .../sszuev/flashcards/android/ui/CommonUI.kt | 7 +- .../flashcards/android/ui/StageOptionsUI.kt | 122 ++++++---- .../flashcards/android/ui/StageShowUI.kt | 4 + android/gradle/libs.versions.toml | 2 + 8 files changed, 207 insertions(+), 150 deletions(-) diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 914e85a..1684579 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -7,7 +7,7 @@ plugins { android { namespace = "com.github.sszuev.flashcards.android" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "com.github.sszuev.flashcards.android" @@ -54,6 +54,7 @@ dependencies { implementation(libs.androidx.splashscreen) implementation(libs.androidx.browser) implementation(libs.androidx.compose.icons.extended) + implementation(libs.androidx.media3) implementation(libs.ktor.client.android) implementation(libs.ktor.client.content.negotiation) implementation(libs.ktor.serialization.kotlinx.json) diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/MainActivity.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/MainActivity.kt index 4000f13..72ff40a 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/MainActivity.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/MainActivity.kt @@ -40,6 +40,7 @@ class MainActivity : ComponentActivity() { } private val cardViewModel: CardViewModel by viewModels { CardsViewModelFactory( + context = application, cardsRepository = CardsRepository(AppConfig.serverUri), ttsRepository = TTSRepository(AppConfig.serverUri), translationRepository = TranslationRepository(AppConfig.serverUri), diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/models/CardViewModel.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/models/CardViewModel.kt index 084bdaf..11396a2 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/models/CardViewModel.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/models/CardViewModel.kt @@ -1,6 +1,6 @@ package com.github.sszuev.flashcards.android.models -import android.media.MediaPlayer +import android.app.Application import android.util.Log import android.util.LruCache import androidx.compose.runtime.State @@ -9,7 +9,10 @@ import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.mutableStateOf import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope -import com.github.sszuev.flashcards.android.PLAY_AUDIO_EMERGENCY_TIMEOUT_MS +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackException +import androidx.media3.common.Player +import androidx.media3.exoplayer.ExoPlayer import com.github.sszuev.flashcards.android.entities.CardEntity import com.github.sszuev.flashcards.android.repositories.CardsRepository import com.github.sszuev.flashcards.android.repositories.InvalidTokenException @@ -28,6 +31,7 @@ import kotlin.io.path.deleteIfExists import kotlin.io.path.outputStream class CardViewModel( + private val context: Application, private val cardsRepository: CardsRepository, private val ttsRepository: TTSRepository, private val translationRepository: TranslationRepository, @@ -49,21 +53,6 @@ class CardViewModel( val sortField: State = _sortField private val _isAscending = mutableStateOf(true) val isAscending: State = _isAscending - - @Suppress("PrivatePropertyName") - private val NULL_AUDIO_MARKER = ByteArray(0) - private val _audioResources = object : LruCache(1024) { - fun getNullable(key: String): ByteArray? { - return when (val result = get(key)) { - NULL_AUDIO_MARKER -> null - else -> result - } - } - - fun putNullable(key: String, value: ByteArray?) { - put(key, value ?: NULL_AUDIO_MARKER) - } - } private val _isAudioLoading = mutableStateMapOf() private val _isAudioPlaying = mutableStateMapOf() @@ -93,6 +82,37 @@ class CardViewModel( private val _additionalCardsDeck = mutableStateOf>(emptyList()) val additionalCardsDeck: State> = _additionalCardsDeck + private val activeMediaPlayers = mutableMapOf() + + private val _audioResources = object : LruCache(1024) { + private val NULL_AUDIO_MARKER = ByteArray(0) + + @Synchronized + fun getNullable(key: String): ByteArray? { + return when (val result = get(key)) { + NULL_AUDIO_MARKER -> null + else -> result + } + } + + @Synchronized + fun putNullable(key: String, value: ByteArray?) { + put(key, value ?: NULL_AUDIO_MARKER) + } + + @Synchronized + fun hasKey(key: String): Boolean { + return super.get(key) != null + } + + @Suppress("ReplaceArrayEqualityOpWithArraysEquals") + @Synchronized + fun hasKeyAndValue(key: String): Boolean { + val res = super.get(key) + return res != null && res != NULL_AUDIO_MARKER + } + } + val selectedCard: CardEntity? get() = if (_selectedCardId.value == null) null else { _cards.value.singleOrNull { it.cardId == _selectedCardId.value } @@ -284,7 +304,7 @@ class CardViewModel( random = true, unknown = true, length = length, - ) + ).distinct() }.map { it.toCardEntity() } if (cards.isEmpty()) { _errorMessage.value = "No cards available in the selected dictionaries." @@ -316,7 +336,7 @@ class CardViewModel( random = true, unknown = false, length = length, - ) + ).distinct() }.map { it.toCardEntity() } if (cards.isEmpty()) { _errorMessage.value = "No cards available in the selected dictionaries." @@ -333,6 +353,15 @@ class CardViewModel( } } + fun waitForAudioToFinish(cardId: String, onComplete: () -> Unit) { + viewModelScope.launch { + while (isAudioLoading(cardId) || isAudioPlaying(cardId)) { + delay(100) + } + onComplete() + } + } + fun loadAndPlayAudio(card: CardEntity) { val cardId = checkNotNull(card.cardId) if (isAudioPlaying(cardId)) { @@ -342,6 +371,7 @@ class CardViewModel( ) return } + stopAllAudio() if (isAudioLoaded(cardId)) { playAudio(card) } else { @@ -351,20 +381,9 @@ class CardViewModel( } } - fun waitForAudioToFinish(cardId: String, onComplete: () -> Unit) { - viewModelScope.launch { - while (isAudioOperationInProgress(cardId)) { - delay(100) - } - onComplete() - } - } - private fun loadAudio(cardId: String, audioResourceId: String, onLoaded: () -> Unit) { - if (_audioResources.get(cardId) != null) { - if (_audioResources.getNullable(cardId) != null) { - onLoaded() - } + if (_audioResources.hasKeyAndValue(cardId)) { + onLoaded() return } viewModelScope.launch { @@ -398,14 +417,13 @@ class CardViewModel( private fun playAudio(card: CardEntity) { val cardId = checkNotNull(card.cardId) - val audioData = _audioResources.get(cardId) - @Suppress("ReplaceArrayEqualityOpWithArraysEquals") - if (audioData == null || audioData == NULL_AUDIO_MARKER) { + val audioData = _audioResources.getNullable(cardId) + if (audioData == null) { Log.d(tag, "playAudion: no audio data for [cardId = $cardId (${card.word})]") return } - if (_isAudioPlaying.putIfAbsent(cardId, true) == true) { + if (setAudioIsPlaying(cardId)) { Log.d(tag, "playAudion: audio is already playing for [cardId = $cardId (${card.word})]") return } @@ -428,63 +446,40 @@ class CardViewModel( } withContext(Dispatchers.Main) { - var isCompleted = false - MediaPlayer().apply { - val timeoutJob = viewModelScope.launch { - delay(PLAY_AUDIO_EMERGENCY_TIMEOUT_MS) - if (!isCompleted && !isPlaying) { - Log.w(tag, "playAudion: timeout reached. Stopping playback.") - releaseAudioResources(cardId, tempFile) - } - } - setDataSource(tempFile.toFile().absolutePath) - setOnPreparedListener { - try { - if (!isPlaying) { + val exoPlayer = ExoPlayer.Builder(context).build() + + exoPlayer.apply { + val mediaItem = MediaItem.fromUri(tempFile.toUri().toString()) + setMediaItem(mediaItem) + prepare() + playWhenReady = true + + addListener(object : Player.Listener { + override fun onPlaybackStateChanged(playbackState: Int) { + if (playbackState == Player.STATE_ENDED) { Log.d( tag, - "playAudio: starting playback [cardId = $cardId (${card.word})]" + "playAudio: playback completed [cardId = $cardId (${card.word})]" ) - start() + this@apply.onFinish(cardId, tempFile) } - } catch (e: Exception) { - Log.e( - tag, - "playAudion: Error during playback start [cardId = $cardId (${card.word})]:" + - " ${e.localizedMessage}", - e - ) - releaseAudioResources(cardId, tempFile) } - } - setOnCompletionListener { - try { - Log.d( + + override fun onPlayerError(error: PlaybackException) { + Log.e( tag, - "playAudion: completed [cardId = $cardId (${card.word})]" + "playAudio: error occurred [cardId = $cardId (${card.word})]: ${error.message}", + error ) - onFinish() - isCompleted = true - } finally { - releaseAudioResources(cardId, tempFile) - timeoutJob.cancel() - } - } - setOnErrorListener { mp, _, _ -> - try { - Log.d(tag, "playAudion: error, [cardId = $cardId (${card.word})]") - mp.onFinish() - isCompleted = true - true - } finally { - releaseAudioResources(cardId, tempFile) - timeoutJob.cancel() + this@apply.onFinish(cardId, tempFile) } - } - prepareAsync() + }) + + activeMediaPlayers[cardId] = this + Log.d(tag, "playAudio: start playing [cardId = $cardId (${card.word})]") + play() } } - } catch (e: Exception) { Log.e( tag, @@ -496,40 +491,53 @@ class CardViewModel( } } + private fun stopAllAudio() { + activeMediaPlayers.forEach { (cardId, player) -> + Log.d(tag, "stopAudio($cardId)") + try { + player.onFinish(cardId, null) + } catch (e: Exception) { + Log.e(tag, "Error stopping audio for [cardId = $cardId]: ${e.localizedMessage}", e) + } + } + activeMediaPlayers.clear() + } + + + private fun ExoPlayer.onFinish(cardId: String, tempFile: Path?) { + try { + release() + } catch (_: Exception) { + } + releaseAudioResources(cardId, tempFile) + } + private fun releaseAudioResources(cardId: String, tempFile: Path?) { viewModelScope.launch(Dispatchers.Main) { _isAudioPlaying.remove(cardId) + activeMediaPlayers.remove(cardId) + } + viewModelScope.launch(Dispatchers.IO) { tempFile?.deleteIfExists() } } - private fun MediaPlayer.onFinish() { - stop() - reset() - release() + fun isAudioPlaying(cardId: String): Boolean { + return _isAudioPlaying[cardId] ?: false } - private fun isAudioLoaded(cardId: String): Boolean { - return _audioResources.get(cardId) != null + private fun setAudioIsPlaying(cardId: String): Boolean { + return _isAudioPlaying.putIfAbsent(cardId, true) == true } - @Suppress("ReplaceArrayEqualityOpWithArraysEquals") - fun audioIsAvailable(cardId: String): Boolean { - return _audioResources.get(cardId) != NULL_AUDIO_MARKER + private fun isAudioLoaded(cardId: String): Boolean { + return _audioResources.hasKey(cardId) } fun isAudioLoading(cardId: String): Boolean { return _isAudioLoading[cardId] ?: false } - fun isAudioOperationInProgress(cardId: String): Boolean { - return isAudioLoading(cardId) || isAudioPlaying(cardId) - } - - private fun isAudioPlaying(cardId: String): Boolean { - return _isAudioPlaying[cardId] ?: false - } - fun selectCard(cardId: String?) { _selectedCardId.value = cardId } diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/models/ViewModelFactory.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/models/ViewModelFactory.kt index 076c4da..da53f20 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/models/ViewModelFactory.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/models/ViewModelFactory.kt @@ -1,5 +1,6 @@ package com.github.sszuev.flashcards.android.models +import android.app.Application import androidx.lifecycle.ViewModel import androidx.lifecycle.ViewModelProvider import com.github.sszuev.flashcards.android.repositories.CardsRepository @@ -23,6 +24,7 @@ class DictionariesViewModelFactory( } class CardsViewModelFactory( + private val context: Application, private val cardsRepository: CardsRepository, private val ttsRepository: TTSRepository, private val translationRepository: TranslationRepository, @@ -32,7 +34,13 @@ class CardsViewModelFactory( @Suppress("UNCHECKED_CAST") override fun create(modelClass: Class): T { if (modelClass.isAssignableFrom(CardViewModel::class.java)) { - return CardViewModel(cardsRepository, ttsRepository, translationRepository, signOut) as T + return CardViewModel( + context = context, + cardsRepository = cardsRepository, + ttsRepository = ttsRepository, + translationRepository = translationRepository, + signOut = signOut, + ) as T } throw IllegalArgumentException("Unknown ViewModel class") } diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/CommonUI.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/CommonUI.kt index 34961a5..b715ae1 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/CommonUI.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/CommonUI.kt @@ -462,11 +462,16 @@ fun AudioPlayerIcon( ) { val cardId = checkNotNull(card.cardId) + val isAudioPlaying = viewModel.isAudioPlaying(cardId) + val isAudioLoading = viewModel.isAudioLoading(cardId) + + val enabled = !(isAudioLoading || isAudioPlaying) + IconButton( onClick = { viewModel.loadAndPlayAudio(card) }, - enabled = viewModel.audioIsAvailable(cardId) && !viewModel.isAudioOperationInProgress(cardId), + enabled = enabled, modifier = modifier.padding(start = 8.dp) ) { if (viewModel.isAudioLoading(cardId)) { diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageOptionsUI.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageOptionsUI.kt index 751d5c7..d4b1366 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageOptionsUI.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageOptionsUI.kt @@ -90,6 +90,12 @@ fun OptionsPanelDirect( onNextStage: () -> Unit, direct: Boolean, ) { + val hasNavigated = remember { mutableStateOf(false) } + + if (hasNavigated.value) { + return + } + val settings = checkNotNull(settingsViewModel.settings.value) { "no settings" } val leftCards = cardViewModel.unknownDeckCards { id -> dictionaryViewModel.dictionaryById(id).numberOfRightAnswers @@ -121,78 +127,77 @@ fun OptionsPanelDirect( } if (cardViewModel.additionalCardsDeck.value.size <= 1) { + Log.d(tag, "onNextStage: not enough additional cards") onNextStage() return } - val cardsMap = remember { - leftCards.associateWith { leftCard -> + val cardsMap = remember { mutableStateOf>>(emptyMap()) } + + LaunchedEffect(Unit) { + val newMap = leftCards.associateWith { leftCard -> val rightCards = cardViewModel.additionalCardsDeck.value.shuffled() .take(settings.stageOptionsNumberOfVariants - 1) - (rightCards + leftCard).distinct().shuffled() - }.toMutableMap() + (rightCards + leftCard).distinctBy { it.cardId }.shuffled() + } + Log.d(tag, "=".repeat(42)) + newMap.forEach { (card, options) -> + Log.d(tag, "${card.cardId} => ${options.map { it.cardId }}") + } + cardsMap.value = newMap } val currentCard = remember { mutableStateOf(leftCards.firstOrNull()) } val selectedOption = remember { mutableStateOf(null) } val isCorrect = remember { mutableStateOf(null) } - if (direct) { - LaunchedEffect(currentCard.value) { - if (currentCard.value == null) { - onNextStage() - return@LaunchedEffect - } - currentCard.value?.let { card -> - Log.d(tag, "Playing audio for: ${card.word}") - cardViewModel.loadAndPlayAudio(card) - } - } - } - fun match(selectedItem: CardEntity): Boolean { return selectedItem.cardId == currentCard.value?.cardId } + fun resetState() { + currentCard.value = null + cardsMap.value = emptyMap() + hasNavigated.value = true + } + fun onOptionSelected(selectedItem: CardEntity) { selectedOption.value = selectedItem isCorrect.value = match(selectedItem) if (!direct) { - Log.d(tag, "Playing audio for: ${selectedItem.word}") + Log.d(tag, "reverse: playing audio for: [${selectedItem.cardId}: ${selectedItem.word}]") cardViewModel.loadAndPlayAudio(selectedItem) } cardViewModel.viewModelScope.launch { delay(STAGE_OPTIONS_CELL_DELAY_MS) if (isCorrect.value == true) { - val card = cardsMap.keys.first() - val cardId = checkNotNull(card.cardId) + currentCard.value?.let { cardsMap.value -= it } - val dictionaryId = checkNotNull(card.dictionaryId) { "no dictionary id" } - val dictionary = dictionaryViewModel.dictionaryById(dictionaryId) + if (cardsMap.value.isEmpty()) { + resetState() + Log.d(tag, "onNextStage: no more cards to proceed") + onNextStage() + return@launch + } - currentCard.value = card + val nextCard = cardsMap.value.keys.firstOrNull() - val wrongAnyway = cardViewModel.wrongAnsweredCardDeckIds.value.contains(cardId) - Log.i( - tag, - "Correct answer for card ${cardId}${if (wrongAnyway) ", but it is already marked as wrong" else ""}" - ) + if (nextCard == null) { + resetState() + Log.d(tag, "onNextStage: null next card") + onNextStage() + return@launch + } - cardViewModel.updateDeckCard( - cardId, - dictionary.numberOfRightAnswers - ) + currentCard.value = nextCard - cardsMap.remove(currentCard.value) - currentCard.value = cardsMap.keys.firstOrNull() - selectedOption.value = null - isCorrect.value = null + val cardId = checkNotNull(nextCard.cardId) + val dictionaryId = checkNotNull(nextCard.dictionaryId) + val dictionary = dictionaryViewModel.dictionaryById(dictionaryId) - if (cardsMap.isEmpty()) { - onNextStage() - } + cardViewModel.updateDeckCard(cardId, dictionary.numberOfRightAnswers) } else { val cardId = checkNotNull(currentCard.value?.cardId) Log.i(tag, "Wrong answer for card $cardId") @@ -200,9 +205,39 @@ fun OptionsPanelDirect( isCorrect.value = null cardViewModel.markDeckCardAsWrong(cardId) } + selectedOption.value = null + isCorrect.value = null + } + } + + Log.d(tag, "current word: [${currentCard.value?.cardId}: ${currentCard.value?.word}]") + + val card = currentCard.value + if (card == null) { + Log.d(tag, "onNextStage: currentCard is null") + resetState() + onNextStage() + return + } + + if (direct) { + LaunchedEffect(currentCard.value) { + if (currentCard.value == null) { + Log.d(tag, "onNextStage:LaunchedEffect -- currentCard is null") + resetState() + onNextStage() + return@LaunchedEffect + } + currentCard.value?.let { card -> + Log.d(tag, "direct: playing audio for: [${card.cardId}: ${card.word}]") + cardViewModel.loadAndPlayAudio(card) + } } } + val dictionary = + dictionaryViewModel.dictionaryById(checkNotNull(card.dictionaryId)) + Column( modifier = Modifier .fillMaxSize() @@ -220,13 +255,6 @@ fun OptionsPanelDirect( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.SpaceBetween ) { - val card = currentCard.value - if (card == null) { - onNextStage() - return - } - val dictionary = - dictionaryViewModel.dictionaryById(checkNotNull(card.dictionaryId)) if (direct || isTextShort(card.translationAsString)) { Text( text = if (direct) card.word else card.translationAsString, @@ -284,7 +312,7 @@ fun OptionsPanelDirect( ) { currentCard.value?.let { leftCard -> items( - cardsMap[leftCard] ?: emptyList(), + cardsMap.value[leftCard] ?: emptyList(), key = { checkNotNull(it.cardId) }) { option -> if (direct) { TableCellTranslation( diff --git a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageShowUI.kt b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageShowUI.kt index 8a6ef09..f378f96 100644 --- a/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageShowUI.kt +++ b/android/app/src/main/java/com/github/sszuev/flashcards/android/ui/StageShowUI.kt @@ -86,6 +86,10 @@ fun StageShowScreen( var currentCard by remember { mutableStateOf(cards.firstOrNull()) } LaunchedEffect(currentCard) { + if (currentCard == null) { + onNextStage() + return@LaunchedEffect + } currentCard?.let { card -> Log.d(tag, "Playing audio for: ${card.word}") cardViewModel.loadAndPlayAudio(card) diff --git a/android/gradle/libs.versions.toml b/android/gradle/libs.versions.toml index bd8cc8e..e2b5339 100644 --- a/android/gradle/libs.versions.toml +++ b/android/gradle/libs.versions.toml @@ -14,6 +14,7 @@ openIdVersion = "0.11.1" splashscreenVersion = "1.0.1" browserVersion = "1.8.0" constraintlayoutVersion = "2.2.0" +media3Version = "1.5.1" [libraries] androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } @@ -35,6 +36,7 @@ androidx-material3 = { group = "androidx.compose.material3", name = "material3" androidx-splashscreen = { group = "androidx.core", name = "core-splashscreen", version.ref = "splashscreenVersion" } androidx-browser = { group = "androidx.browser", name = "browser", version.ref = "browserVersion" } androidx-constraintlayout = { group = "androidx.constraintlayout", name = "constraintlayout", version.ref = "constraintlayoutVersion" } +androidx-media3 = { group = "androidx.media3", name = "media3-exoplayer", version.ref = "media3Version" } ktor-client-android = { group = "io.ktor", name = "ktor-client-android", version.ref = "ktorVersion" } ktor-client-content-negotiation = { group = "io.ktor", name = "ktor-client-content-negotiation", version.ref = "ktorVersion" } ktor-serialization-kotlinx-json = { group = "io.ktor", name = "ktor-serialization-kotlinx-json", version.ref = "ktorVersion" }