Merge pull request #81 from android/gemini-multimodal

Gemini API multimodal support
android · Jun 27, 2024 · 7a62784 · 7a62784
2 parents 3e55d4f + 147d6ed
commit 7a62784
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 1 deletion.
diff --git a/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt b/app/src/main/java/com/google/android/samples/socialite/repository/ChatRepository.kt
@@ -17,6 +17,8 @@
 package com.google.android.samples.socialite.repository
 
 import android.content.Context
+import android.graphics.BitmapFactory
+import android.net.Uri
 import android.widget.Toast
 import androidx.datastore.core.DataStore
 import androidx.datastore.preferences.core.Preferences
@@ -114,7 +116,21 @@ class ChatRepository @Inject internal constructor(
 
                 // Send a message prompt to the model to generate a response
                 var response = try {
-                    chat.sendMessage(text).text?.trim() ?: "..."
+                    if (mediaMimeType?.contains("image") == true) {
+                        appContext.contentResolver.openInputStream(
+                            Uri.parse(mediaUri),
+                        ).use {
+                            if (it != null) {
+                                chat.sendMessage(BitmapFactory.decodeStream(it)).text?.trim() ?: "..."
+                            } else {
+                                appContext.getString(
+                                    R.string.image_error,
+                                )
+                            }
+                        }
+                    } else {
+                        chat.sendMessage(text).text?.trim() ?: "..."
+                    }
                 } catch (e: Exception) {
                     e.printStackTrace()
                     appContext.getString(

diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
@@ -70,6 +70,7 @@
     <string name="ai_chatbot_setting_disabled">disabled</string>
     <string name="set_api_key_toast">Please set your API key in local.properties</string>
     <string name="gemini_error">Gemini error (%1$s)</string>
+    <string name="image_error">Can\'t open image</string>
     <string name="unknown_error">Unknown error</string>
 
 </resources>