diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 7e327172..6d535d75 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -22,6 +22,8 @@ + + diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt new file mode 100644 index 00000000..1a125f0c --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiLiveAudioHelper.kt @@ -0,0 +1,233 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data + +import android.annotation.SuppressLint +import android.media.AudioAttributes +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.AudioTrack +import android.media.MediaRecorder +import android.media.audiofx.AcousticEchoCanceler +import android.util.Log +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Manages low-latency PCM audio recording (16 kHz mono 16-bit) and playback (24 kHz mono 16-bit) + * for a Gemini Live bidirectional audio session. + */ +class GeminiLiveAudioHelper { + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val isReleased = AtomicBoolean(false) + private val isRecordingPaused = AtomicBoolean(false) + + private var audioRecord: AudioRecord? = null + private var echoCanceler: AcousticEchoCanceler? = null + private var audioTrack: AudioTrack? = null + private var playbackJob: Job? = null + + private val audioQueue = ConcurrentLinkedQueue() + + @SuppressLint("MissingPermission") + fun start() { + val minRecordBuf = + AudioRecord.getMinBufferSize( + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val recorder = + AudioRecord( + MediaRecorder.AudioSource.VOICE_COMMUNICATION, + INPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minRecordBuf, + ) + if (recorder.state != AudioRecord.STATE_INITIALIZED) { + recorder.release() + throw IllegalStateException("Failed to initialize AudioRecord") + } + + if (AcousticEchoCanceler.isAvailable()) { + runCatching { + echoCanceler = + AcousticEchoCanceler.create(recorder.audioSessionId)?.apply { + enabled = true + } + }.onFailure { e -> + Log.w(TAG, "Failed to enable AcousticEchoCanceler", e) + } + } + + val minTrackBuf = + AudioTrack.getMinBufferSize( + OUTPUT_SAMPLE_RATE, + AudioFormat.CHANNEL_OUT_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ).coerceAtLeast(DEFAULT_READ_BUFFER_SIZE) + + val track = + AudioTrack.Builder() + .setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build(), + ) + .setAudioFormat( + AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setSampleRate(OUTPUT_SAMPLE_RATE) + .setChannelMask(AudioFormat.CHANNEL_OUT_MONO) + .build(), + ) + .setBufferSizeInBytes(minTrackBuf) + .setTransferMode(AudioTrack.MODE_STREAM) + .build() + + track.play() + recorder.startRecording() + + audioRecord = recorder + audioTrack = track + + playbackJob = + scope.launch { + while (isActive && !isReleased.get()) { + val chunk = audioQueue.poll() + if (chunk != null) { + val currentTrack = audioTrack + if (currentTrack != null && !isReleased.get()) { + currentTrack.write(chunk, 0, chunk.size) + } + } else { + delay(PLAYBACK_POLL_DELAY_MS) + } + } + } + } + + /** Emits PCM 16-bit 16 kHz mono byte chunks recorded from the microphone. */ + fun listenToRecording(): Flow = + flow { + val buffer = ByteArray(DEFAULT_READ_BUFFER_SIZE) + while (currentCoroutineContext().isActive && !isReleased.get()) { + if (isRecordingPaused.get()) { + delay(RECORD_PAUSE_DELAY_MS) + continue + } + val recorder = audioRecord ?: break + val bytesRead = recorder.read(buffer, 0, buffer.size) + if (bytesRead > 0 && !isRecordingPaused.get()) { + emit(buffer.copyOf(bytesRead)) + } else if (bytesRead < 0) { + Log.w(TAG, "AudioRecord.read returned error code: $bytesRead") + break + } + } + }.flowOn(Dispatchers.IO) + + fun queueAudio(data: ByteArray) { + if (!isReleased.get()) { + audioQueue.add(data) + } + } + + /** Clears any queued model audio when the user interrupts the model. */ + fun stopAudio() { + audioQueue.clear() + } + + fun pauseRecording() { + isRecordingPaused.set(true) + runCatching { + val recorder = audioRecord + if (recorder?.recordingState == AudioRecord.RECORDSTATE_RECORDING) { + recorder.stop() + } + } + } + + fun resumeRecording() { + if (!isReleased.get()) { + runCatching { + val recorder = audioRecord + if (recorder?.state == AudioRecord.STATE_INITIALIZED && + recorder.recordingState != AudioRecord.RECORDSTATE_RECORDING + ) { + recorder.startRecording() + } + } + isRecordingPaused.set(false) + } + } + + fun release() { + if (!isReleased.compareAndSet(false, true)) return + audioQueue.clear() + playbackJob?.cancel() + scope.cancel() + + runCatching { echoCanceler?.release() } + echoCanceler = null + + runCatching { + audioRecord?.let { recorder -> + if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) { + recorder.stop() + } + recorder.release() + } + } + audioRecord = null + + runCatching { + audioTrack?.let { track -> + if (track.playState == AudioTrack.PLAYSTATE_PLAYING) { + track.pause() + track.flush() + track.stop() + } + track.release() + } + } + audioTrack = null + } + + companion object { + private const val TAG = "GeminiLiveAudioHelper" + private const val INPUT_SAMPLE_RATE = 16000 + private const val OUTPUT_SAMPLE_RATE = 24000 + private const val DEFAULT_READ_BUFFER_SIZE = 2048 + private const val PLAYBACK_POLL_DELAY_MS = 10L + private const val RECORD_PAUSE_DELAY_MS = 25L + } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index cc109813..e195b146 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -45,6 +45,40 @@ class GeminiToolConverter return convertToolToGeminiSchema(tool) } + /** + * Converts an [AppFunctionMetadata] into a Gemini Live (`BidiGenerateContent`) + * `FunctionDeclaration` JSON object, normalizing OpenAPI schema fields. + */ + fun convertForLiveApi(tool: AppFunctionMetadata): JsonObject { + val baseSchema = convertToolToGeminiSchema(tool) + return normalizeForLiveApi(baseSchema) + } + + private fun normalizeForLiveApi(obj: JsonObject): JsonObject { + val typeValue = (obj[KEY_TYPE] as? JsonPrimitive)?.content + return buildJsonObject { + for ((key, value) in obj) { + when { + key == KEY_FORMAT && + (value as? JsonPrimitive)?.content == VALUE_FILE_REFERENCE -> { + // Omit Interactions-API-specific file_reference format in BidiGenerateContent + } + key == KEY_ENUMS -> { + if (typeValue == VALUE_STRING) { + put(KEY_ENUM_LIVE, value) + } + } + value is JsonObject -> { + put(key, normalizeForLiveApi(value)) + } + else -> { + put(key, value) + } + } + } + } + } + override fun getToolName(tool: AppFunctionMetadata): String { return getGeminiFunctionName(tool) } @@ -295,6 +329,7 @@ class GeminiToolConverter private const val VALUE_ARRAY = "array" private const val KEY_ITEMS = "items" private const val KEY_ENUMS = "enums" + private const val KEY_ENUM_LIVE = "enum" private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") private const val MAX_NAME_LENGTH = 64 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 3757f9df..347257d8 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -38,6 +38,7 @@ import io.ktor.client.HttpClient import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.websocket.WebSockets import io.ktor.serialization.kotlinx.json.json import kotlinx.serialization.json.Json import javax.inject.Singleton @@ -109,6 +110,7 @@ abstract class DataModule { ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } + install(WebSockets) } } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index d2af9ee1..0b3b7a8d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -94,6 +94,15 @@ class AgentOrchestrator /** The current status of the agent. */ val status: StateFlow = _status.asStateFlow() + /** Returns all currently enabled AppFunction and internal tools. */ + suspend fun getEnabledTools(targetPackageName: String? = null): List { + val disconnectedApps = settingsRepository.disconnectedApps.first() + val allTools = + getAppFunctionsUseCase().first().values.flatten() + + agentInternalTools.getInternalToolsMetadata().map { it.first } + return filterTools(allTools, disconnectedApps, targetPackageName) + } + /** * Observes messages for a specific thread and processes any pending agent responses. This * method suspends and collects messages until cancelled. @@ -524,6 +533,236 @@ class AgentOrchestrator return ExecuteToolCallsResult.Success(results) } + /** + * Executes tool calls initiated by a Gemini Live voice session for [threadId], updating + * [status], saving any generated attachments or [PendingIntent] confirmation actions to the + * chat thread, and returning a [ToolOutput] for every call. + */ + suspend fun executeLiveToolCalls( + threadId: String, + toolCalls: List, + tools: List, + ): List { + val outputs = mutableListOf() + val capturedAttachments = mutableListOf() + try { + for (toolCall in toolCalls) { + _status.value = + AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + + if (toolCall.packageName == AgentInternalTools.INTERNAL_TOOL_PACKAGE) { + val executionResult = executeInternalTool(toolCall) + executionResult + .onSuccess { jsonString -> + runCatching { + val json = JSONObject(jsonString) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType").takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType), + ) + } + } + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = jsonString, + ), + ) + } + .onFailure { exception -> + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Internal tool execution failed: ${exception.message}", + ) + }.toString(), + ), + ) + } + continue + } + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId + } + + if (matchingTool == null) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put("error", "Tool not found: ${toolCall.functionId}") + }.toString(), + ), + ) + continue + } + + @Suppress("UNCHECKED_CAST") + val rawConvertedInputs = + toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: Exception) { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Failed to download remote file reference: ${e.message}", + ) + }.toString(), + ), + ) + continue + } + + grantContentUriPermissionsRecursively( + context, + toolCall.packageName, + convertedInputs, + ) + + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs, + ) + } + if (appFunctionDataResult.isFailure) { + val errMsg = appFunctionDataResult.exceptionOrNull()?.message + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put( + "error", + "Failed to convert arguments for ${toolCall.functionId}: $errMsg", + ) + }.toString(), + ), + ) + continue + } + + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = threadId, + ) + + when (executionResult) { + is ExecuteAppFunctionResult.Data -> { + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = executionResult.formattedJson, + ), + ) + } + + is ExecuteAppFunctionResult.PendingIntentAction -> { + val pendingIntentId = UUID.randomUUID().toString() + savePendingIntentUseCase(pendingIntentId, executionResult.pendingIntent) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = pendingIntentId, + ) + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + JSONObject().apply { + put("status", "confirmation_required") + put( + "message", + "A confirmation button has been displayed in the chat. Ask the user to tap it to confirm.", + ) + }.toString(), + ), + ) + } + + is ExecuteAppFunctionResult.Error -> { + val exception = executionResult.exception + if (exception is CancellationException) { + throw exception + } + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) + val errorJson = + if (appFunctionException != null) { + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId, + ) + } else { + JSONObject().apply { + put( + "error", + "Tool execution failed for ${toolCall.functionId}: ${exception.message}", + ) + }.toString() + } + outputs.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = errorJson, + ), + ) + } + } + } + + if (capturedAttachments.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "", + processingStatus = MessageProcessingStatus.PROCESSED, + attachments = capturedAttachments, + ) + } + } finally { + _status.value = AgentStatus.Idle + } + return outputs + } + private suspend fun completeMessageWithError( messageId: String, threadId: String, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt new file mode 100644 index 00000000..997dc03f --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManager.kt @@ -0,0 +1,608 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import android.util.Base64 +import android.util.Log +import androidx.appfunctions.metadata.AppFunctionMetadata +import com.example.appfunctions.agent.data.GeminiLiveAudioHelper +import com.example.appfunctions.agent.data.GeminiToolConverter +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus +import com.example.appfunctions.agent.data.db.entities.MessageRole +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import io.ktor.client.HttpClient +import io.ktor.client.plugins.websocket.DefaultClientWebSocketSession +import io.ktor.client.plugins.websocket.webSocket +import io.ktor.websocket.CloseReason +import io.ktor.websocket.Frame +import io.ktor.websocket.close +import io.ktor.websocket.readBytes +import io.ktor.websocket.readText +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.put +import java.time.LocalDate +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Manages a bidirectional voice session with the Gemini Live API (`models/gemini-3.8-live`), + * bridging realtime audio I/O, transcriptions, and Android AppFunction tool execution. + */ +@Singleton +class GeminiLiveSessionManager + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + private val settingsRepository: SettingsRepository, + private val agentOrchestrator: AgentOrchestrator, + private val sendMessageUseCase: SendMessageUseCase, + ) { + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + + private val _state = MutableStateFlow(LiveSessionState.Idle) + + /** Current state of the Gemini Live session. */ + val state: StateFlow = _state.asStateFlow() + + private var sessionJob: Job? = null + private var audioHelper: GeminiLiveAudioHelper? = null + + internal var audioHelperFactory: () -> GeminiLiveAudioHelper = { GeminiLiveAudioHelper() } + + /** Toggles the Gemini Live voice session for [threadId]. */ + fun toggleSession( + threadId: String, + targetPackageName: String? = null, + ) { + when (_state.value) { + is LiveSessionState.Connecting, + is LiveSessionState.Listening, + -> stopSession() + is LiveSessionState.Idle, + is LiveSessionState.Error, + -> startSession(threadId, targetPackageName) + } + } + + /** Starts a new Gemini Live voice session for [threadId]. */ + fun startSession( + threadId: String, + targetPackageName: String? = null, + ) { + stopSessionInternal(resetState = false) + _state.value = LiveSessionState.Connecting + + sessionJob = + scope.launch { + val userTranscriptBuilder = StringBuilder() + val modelTranscriptBuilder = StringBuilder() + try { + val apiKey = + settingsRepository.geminiApiKey.first()?.takeIf { it.isNotBlank() } + if (apiKey == null) { + val errorMsg = "API key is missing for GEMINI" + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED, + ) + _state.value = LiveSessionState.Error(errorMsg) + return@launch + } + + val tools = agentOrchestrator.getEnabledTools(targetPackageName) + val wsUrl = "$GEMINI_LIVE_WS_URL?key=$apiKey" + + httpClient.webSocket(urlString = wsUrl) { + val setupMessage = buildSetupMessage(tools) + Log.d(TAG, "Sending Gemini Live setup for model $LIVE_MODEL_NAME") + send(Frame.Text(setupMessage.toString())) + + val firstFrameText = receiveNextJsonText() + if (firstFrameText == null) { + throw IllegalStateException( + "WebSocket closed before setupComplete was received", + ) + } + val firstJson = Json.parseToJsonElement(firstFrameText).jsonObject + if (!firstJson.containsKey("setupComplete")) { + throw IllegalStateException( + "Unexpected setup response: $firstFrameText", + ) + } + + val helper = + audioHelperFactory().also { + it.start() + audioHelper = it + } + _state.value = LiveSessionState.Listening + + val sendAudioJob = + launch { + helper.listenToRecording().collect { pcmBytes -> + val base64Audio = + Base64.encodeToString(pcmBytes, Base64.NO_WRAP) + val realtimeInput = + buildJsonObject { + put( + "realtimeInput", + buildJsonObject { + put( + "audio", + buildJsonObject { + put( + "mimeType", + "audio/pcm;rate=16000", + ) + put("data", base64Audio) + }, + ) + }, + ) + } + send(Frame.Text(realtimeInput.toString())) + } + } + + try { + while (isActive) { + val messageText = receiveNextJsonText() ?: break + val serverJson = + runCatching { + Json.parseToJsonElement(messageText).jsonObject + }.getOrNull() ?: continue + + handleServerMessage( + serverJson = serverJson, + threadId = threadId, + tools = tools, + helper = helper, + userTranscriptBuilder = userTranscriptBuilder, + modelTranscriptBuilder = modelTranscriptBuilder, + ) + } + } finally { + sendAudioJob.cancel() + runCatching { + close( + CloseReason( + CloseReason.Codes.NORMAL, + "Session ended by client", + ), + ) + } + } + } + + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + _state.value = LiveSessionState.Idle + } catch (e: CancellationException) { + scope.launch { + flushTranscripts( + threadId, + userTranscriptBuilder, + modelTranscriptBuilder, + ) + } + throw e + } catch (e: Exception) { + Log.e(TAG, "Gemini Live session error", e) + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + val errorText = + "Gemini Live error: ${e.message ?: "Connection failed"}" + runCatching { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorText, + processingStatus = MessageProcessingStatus.FAILED, + ) + } + _state.value = LiveSessionState.Error(errorText) + } finally { + audioHelper?.release() + audioHelper = null + } + } + } + + /** Stops any active Gemini Live voice session. */ + fun stopSession() { + stopSessionInternal(resetState = true) + } + + private fun stopSessionInternal(resetState: Boolean) { + sessionJob?.cancel() + sessionJob = null + audioHelper?.release() + audioHelper = null + if (resetState) { + _state.value = LiveSessionState.Idle + } + } + + private suspend fun DefaultClientWebSocketSession.receiveNextJsonText(): String? { + while (isActive) { + val frame = runCatching { incoming.receive() }.getOrNull() ?: return null + when (frame) { + is Frame.Text -> return frame.readText() + is Frame.Binary -> return frame.readBytes().decodeToString() + is Frame.Close -> return null + else -> continue + } + } + return null + } + + private suspend fun DefaultClientWebSocketSession.handleServerMessage( + serverJson: JsonObject, + threadId: String, + tools: List, + helper: GeminiLiveAudioHelper, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + val serverContent = serverJson["serverContent"]?.jsonObject + if (serverContent != null) { + val interrupted = + serverContent["interrupted"]?.jsonPrimitive?.booleanOrNull ?: false + if (interrupted) { + helper.stopAudio() + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + val inputTranscription = + serverContent["inputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!inputTranscription.isNullOrEmpty()) { + userTranscriptBuilder.append(inputTranscription) + } + + val outputTranscription = + serverContent["outputTranscription"] + ?.jsonObject + ?.get("text") + ?.jsonPrimitive + ?.contentOrNull + if (!outputTranscription.isNullOrEmpty()) { + // Flush user transcript before model response transcript starts + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + modelTranscriptBuilder.append(outputTranscription) + } + + val parts = + serverContent["modelTurn"]?.jsonObject?.get("parts")?.jsonArray + if (parts != null) { + for (part in parts) { + val inlineData = part.jsonObject["inlineData"]?.jsonObject ?: continue + val mimeType = + inlineData["mimeType"]?.jsonPrimitive?.contentOrNull ?: "" + val base64Data = + inlineData["data"]?.jsonPrimitive?.contentOrNull ?: continue + if (mimeType.startsWith("audio/pcm")) { + val pcmBytes = Base64.decode(base64Data, Base64.DEFAULT) + helper.queueAudio(pcmBytes) + } + } + } + + val turnComplete = + serverContent["turnComplete"]?.jsonPrimitive?.booleanOrNull ?: false + if (turnComplete) { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + } + } + + val toolCallObj = serverJson["toolCall"]?.jsonObject + if (toolCallObj != null) { + flushTranscripts(threadId, userTranscriptBuilder, modelTranscriptBuilder) + val functionCalls = toolCallObj["functionCalls"]?.jsonArray ?: JsonArray(emptyList()) + if (functionCalls.isNotEmpty()) { + helper.pauseRecording() + try { + val parsedCalls = mutableListOf() + val unknownCalls = mutableListOf>() + + for (callElement in functionCalls) { + val callObj = callElement.jsonObject + val callId = callObj["id"]?.jsonPrimitive?.contentOrNull ?: "" + val name = callObj["name"]?.jsonPrimitive?.contentOrNull ?: continue + val matchingTool = + tools.find { tool -> toolConverter.getToolName(tool) == name } + + if (matchingTool == null) { + unknownCalls.add(callId to name) + continue + } + + val argsObj = callObj["args"]?.jsonObject + val argumentsMap = mutableMapOf() + if (argsObj != null) { + for ((key, value) in argsObj) { + argumentsMap[key] = value.toPrimitive() + } + } + parsedCalls.add( + LlmResponsePart.ToolCall( + packageName = matchingTool.packageName, + functionId = matchingTool.id, + arguments = argumentsMap, + callId = callId, + ), + ) + } + + val toolOutputs = + if (parsedCalls.isNotEmpty()) { + agentOrchestrator.executeLiveToolCalls( + threadId = threadId, + toolCalls = parsedCalls, + tools = tools, + ) + } else { + emptyList() + } + + val toolResponseMessage = + buildToolResponseMessage( + toolOutputs = toolOutputs, + unknownCalls = unknownCalls, + tools = tools, + ) + Log.d(TAG, "Sending Gemini Live toolResponse: $toolResponseMessage") + send(Frame.Text(toolResponseMessage.toString())) + } finally { + helper.resumeRecording() + } + } + } + } + + private suspend fun flushTranscripts( + threadId: String, + userTranscriptBuilder: StringBuilder, + modelTranscriptBuilder: StringBuilder, + ) { + flushTranscript(threadId, MessageRole.USER, userTranscriptBuilder) + flushTranscript(threadId, MessageRole.ASSISTANT, modelTranscriptBuilder) + } + + private suspend fun flushTranscript( + threadId: String, + role: MessageRole, + builder: StringBuilder, + ) { + val text = builder.toString().trim() + builder.clear() + if (text.isNotEmpty()) { + sendMessageUseCase( + threadId = threadId, + role = role, + textContent = text, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + } + + internal fun buildSetupMessage(tools: List): JsonObject { + val functionDeclarations = + tools + .sortedByDescending { it.id.startsWith(it.packageName) } + .distinctBy { toolConverter.getToolName(it) } + .mapNotNull { tool -> + try { + toolConverter.convertForLiveApi(tool) + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null + } + } + + return buildJsonObject { + put( + "setup", + buildJsonObject { + put("model", LIVE_MODEL_NAME) + put( + "generationConfig", + buildJsonObject { + put( + "responseModalities", + buildJsonArray { add(JsonPrimitive("AUDIO")) }, + ) + put( + "speechConfig", + buildJsonObject { + put( + "voiceConfig", + buildJsonObject { + put( + "prebuiltVoiceConfig", + buildJsonObject { + put("voiceName", DEFAULT_VOICE_NAME) + }, + ) + }, + ) + }, + ) + }, + ) + put( + "systemInstruction", + buildJsonObject { + put( + "parts", + buildJsonArray { + add( + buildJsonObject { + put("text", getSystemInstruction()) + }, + ) + }, + ) + }, + ) + if (functionDeclarations.isNotEmpty()) { + put( + "tools", + buildJsonArray { + add( + buildJsonObject { + put( + "functionDeclarations", + JsonArray(functionDeclarations), + ) + }, + ) + }, + ) + } + put("inputAudioTranscription", buildJsonObject {}) + put("outputAudioTranscription", buildJsonObject {}) + }, + ) + } + } + + private fun buildToolResponseMessage( + toolOutputs: List, + unknownCalls: List>, + tools: List, + ): JsonObject { + return buildJsonObject { + put( + "toolResponse", + buildJsonObject { + put( + "functionResponses", + buildJsonArray { + for (output in toolOutputs) { + val matchingTool = tools.find { it.id == output.functionId } + val mappedName = + if (matchingTool != null) { + toolConverter.getToolName(matchingTool) + } else { + output.functionId + } + val parsedResult = + runCatching { + Json.parseToJsonElement(output.result) + }.getOrElse { + JsonPrimitive(output.result) + } + add( + buildJsonObject { + if (output.callId.isNotEmpty()) { + put("id", output.callId) + } + put("name", mappedName) + put( + "response", + buildJsonObject { + put("result", parsedResult) + }, + ) + }, + ) + } + for ((callId, unknownName) in unknownCalls) { + add( + buildJsonObject { + if (callId.isNotEmpty()) { + put("id", callId) + } + put("name", unknownName) + put( + "response", + buildJsonObject { + put( + "error", + "Unknown function: $unknownName", + ) + }, + ) + }, + ) + } + }, + ) + }, + ) + } + } + + private fun JsonElement.toPrimitive(): Any? { + return when (this) { + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } + } + } + + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI voice assistant running on Android using Gemini Live. Today's date is $currentDate. + Always reply to the user using concise, friendly, natural spoken language. + You have access to Android AppFunctions and internal device tools. Use them whenever the user asks to perform an action or look up information on their device. + When a user asks you to generate an image, call the generateImage tool and then confirm to the user in a natural sentence. + When a tool returns status "confirmation_required", let the user know that a confirmation button is on their screen and ask them to tap it to proceed. + """.trimIndent() + } + + companion object { + private const val TAG = "GeminiLiveSessionMgr" + const val LIVE_MODEL_NAME = "models/gemini-3.8-live" + private const val DEFAULT_VOICE_NAME = "CHARON" + private const val GEMINI_LIVE_WS_URL = + "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + } + } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt new file mode 100644 index 00000000..f9fc51c0 --- /dev/null +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/LiveSessionState.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +/** Represents the state of a Gemini Live voice session. */ +sealed class LiveSessionState { + object Idle : LiveSessionState() + + object Connecting : LiveSessionState() + + object Listening : LiveSessionState() + + data class Error(val message: String) : LiveSessionState() +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index b1c9f3b3..4ab33cd3 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -15,7 +15,10 @@ */ package com.example.appfunctions.agent.ui.screens.agentdemo +import android.Manifest import android.content.pm.PackageManager +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.expandHorizontally import androidx.compose.animation.shrinkHorizontally @@ -49,6 +52,8 @@ import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material.icons.filled.ArrowDropDown import androidx.compose.material.icons.filled.Menu +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.Stop import androidx.compose.material.icons.filled.Warning import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Card @@ -72,6 +77,7 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.rememberDrawerState import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -116,6 +122,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.window.Popup import androidx.compose.ui.window.PopupPositionProvider import androidx.compose.ui.window.PopupProperties +import androidx.core.content.ContextCompat import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle @@ -127,6 +134,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.ui.screens.debugging.LazyExposedDropdownMenu import com.mikepenz.markdown.m3.Markdown @@ -137,6 +145,12 @@ import kotlinx.coroutines.launch fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { val uiState by viewModel.uiState.collectAsStateWithLifecycle() + DisposableEffect(Unit) { + onDispose { + viewModel.onEvent(AgentUiEvent.OnStopLiveSession) + } + } + AgentDemoContent(uiState = uiState, onEvent = viewModel::onEvent) } @@ -230,6 +244,44 @@ fun AgentDemoLoadedScreen( var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } var selectedAppPackageName by remember { mutableStateOf(null) } + val context = LocalContext.current + val audioPermissionLauncher = + rememberLauncherForActivityResult( + contract = ActivityResultContracts.RequestPermission(), + ) { isGranted -> + if (isGranted) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } + } + + val toggleLiveSession = { + val isActive = + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + if (isActive) { + onEvent(AgentUiEvent.OnStopLiveSession) + } else { + val hasRecordAudioPermission = + try { + ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO, + ) == PackageManager.PERMISSION_GRANTED + } catch (_: Exception) { + true + } + if (hasRecordAudioPermission) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } else { + try { + audioPermissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } catch (_: IllegalStateException) { + onEvent(AgentUiEvent.OnToggleLiveSession(selectedAppPackageName)) + } + } + } + } + val chipBgColor = MaterialTheme.colorScheme.primaryContainer val chipTextColor = MaterialTheme.colorScheme.onPrimaryContainer val visualTransformation = @@ -251,6 +303,7 @@ fun AgentDemoLoadedScreen( .weight(1f) .padding(horizontal = 8.dp), currentThread = uiState.currentThread, + liveSessionState = uiState.liveSessionState, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { if (isWideScreen) { @@ -324,6 +377,18 @@ fun AgentDemoLoadedScreen( } } + if ( + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + ) { + item { + LiveSessionIndicator( + liveSessionState = uiState.liveSessionState, + onStopClick = { onEvent(AgentUiEvent.OnStopLiveSession) }, + ) + } + } + items( items = uiState.messages.reversed(), key = { message -> message.messageId }, @@ -440,17 +505,49 @@ fun AgentDemoLoadedScreen( focusedBorderColor = Color.Transparent, ), trailingIcon = { - IconButton( - onClick = sendMessage, - enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle, + val isLiveActive = + uiState.liveSessionState is LiveSessionState.Connecting || + uiState.liveSessionState is LiveSessionState.Listening + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.padding(end = 4.dp), ) { - Icon( - imageVector = Icons.AutoMirrored.Filled.Send, - contentDescription = - stringResource(R.string.agent_demo_send), - ) + IconButton( + onClick = toggleLiveSession, + ) { + Icon( + imageVector = + if (isLiveActive) { + Icons.Default.Stop + } else { + Icons.Default.Mic + }, + contentDescription = + if (isLiveActive) { + stringResource(R.string.agent_demo_live_stop) + } else { + stringResource(R.string.agent_demo_live_start) + }, + tint = + if (isLiveActive) { + MaterialTheme.colorScheme.error + } else { + MaterialTheme.colorScheme.primary + }, + ) + } + IconButton( + onClick = sendMessage, + enabled = + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle, + ) { + Icon( + imageVector = Icons.AutoMirrored.Filled.Send, + contentDescription = + stringResource(R.string.agent_demo_send), + ) + } } }, ) @@ -524,14 +621,29 @@ fun AgentDemoLoadedScreen( fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, + liveSessionState: LiveSessionState = LiveSessionState.Idle, onModelSelected: (LlmModel) -> Unit, onMenuClick: () -> Unit, ) { var expanded by remember { mutableStateOf(false) } + val isLiveActive = + liveSessionState is LiveSessionState.Connecting || + liveSessionState is LiveSessionState.Listening + + LaunchedEffect(isLiveActive) { + if (isLiveActive) { + expanded = false + } + } + ExposedDropdownMenuBox( modifier = modifier, - expanded = expanded, - onExpandedChange = { expanded = !expanded }, + expanded = expanded && !isLiveActive, + onExpandedChange = { + if (!isLiveActive) { + expanded = !expanded + } + }, ) { Surface( modifier = Modifier.padding(bottom = 8.dp), @@ -540,13 +652,17 @@ fun ModelDropdown( color = MaterialTheme.colorScheme.surfaceBright, ) { val text = - currentThread?.llmModel?.modelName - ?: stringResource(R.string.agent_demo_select_model_to_create_thread) - val textColor = - if (currentThread != null) { - MaterialTheme.colorScheme.onSurface + if (isLiveActive) { + "gemini-3.8-live" } else { - MaterialTheme.colorScheme.error + currentThread?.llmModel?.modelName + ?: stringResource(R.string.agent_demo_select_model_to_create_thread) + } + val textColor = + when { + isLiveActive -> MaterialTheme.colorScheme.primary + currentThread != null -> MaterialTheme.colorScheme.onSurface + else -> MaterialTheme.colorScheme.error } Row( @@ -567,7 +683,7 @@ fun ModelDropdown( .fillMaxHeight() .menuAnchor( ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true, + enabled = !isLiveActive, ), verticalAlignment = Alignment.CenterVertically, ) { @@ -585,7 +701,9 @@ fun ModelDropdown( overflow = TextOverflow.Ellipsis, ) } - Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) + if (!isLiveActive) { + Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) + } } } } @@ -907,6 +1025,59 @@ fun StatusIndicator( } } +@Composable +fun LiveSessionIndicator( + liveSessionState: LiveSessionState, + onStopClick: () -> Unit, +) { + val label = + when (liveSessionState) { + LiveSessionState.Connecting -> stringResource(R.string.agent_demo_live_connecting) + LiveSessionState.Listening -> stringResource(R.string.agent_demo_live_listening) + else -> return + } + + Surface( + modifier = + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + shape = MaterialTheme.shapes.large, + color = MaterialTheme.colorScheme.secondaryContainer, + shadowElevation = 2.dp, + ) { + Row( + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.weight(1f), + ) { + CircularProgressIndicator( + modifier = Modifier.size(20.dp), + strokeWidth = 2.dp, + color = MaterialTheme.colorScheme.onSecondaryContainer, + ) + Spacer(modifier = Modifier.width(12.dp)) + Text( + text = label, + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSecondaryContainer, + ) + } + IconButton(onClick = onStopClick) { + Icon( + imageVector = Icons.Default.Stop, + contentDescription = stringResource(R.string.agent_demo_live_stop), + tint = MaterialTheme.colorScheme.error, + ) + } + } + } +} + @Composable fun ChatHistorySidePanel( threads: List, diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt index 2f92660d..2212ddb7 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModel.kt @@ -27,6 +27,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.GeminiLiveSessionManager import com.example.appfunctions.agent.domain.appfunction.AppInfo import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase @@ -64,6 +65,7 @@ class AgentDemoViewModel private val consumePendingIntentUseCase: ConsumePendingIntentUseCase, private val getInstalledAppsUseCase: GetInstalledAppsUseCase, private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val geminiLiveSessionManager: GeminiLiveSessionManager, ) : ViewModel() { private val _uiState = MutableStateFlow(AgentUiState.Loading) val uiState: StateFlow = _uiState.asStateFlow() @@ -95,6 +97,15 @@ class AgentDemoViewModel } } + viewModelScope.launch { + geminiLiveSessionManager.state.collect { liveState -> + val currentState = _uiState.value + if (currentState is AgentUiState.Loaded) { + _uiState.value = currentState.copy(liveSessionState = liveState) + } + } + } + viewModelScope.launch { val threads = manageThreadsUseCase.getThreads().first() if (threads.isEmpty()) { @@ -140,10 +151,14 @@ class AgentDemoViewModel activePendingActionIds = currentLoadedState?.activePendingActionIds ?: emptySet(), installedApps = apps, + liveSessionState = geminiLiveSessionManager.state.value, ) // Start observing messages for the current thread if not already doing so if (observationJob == null || previousThreadId != currentThread.threadId) { + if (previousThreadId != null && previousThreadId != currentThread.threadId) { + geminiLiveSessionManager.stopSession() + } observationJob?.cancel() observationJob = viewModelScope.launch { @@ -207,9 +222,25 @@ class AgentDemoViewModel launchPendingIntentUseCase(pendingIntent) } } + is AgentUiEvent.OnToggleLiveSession -> { + if (currentState is AgentUiState.Loaded) { + geminiLiveSessionManager.toggleSession( + threadId = currentState.currentThread.threadId, + targetPackageName = event.targetPackageName, + ) + } + } + AgentUiEvent.OnStopLiveSession -> { + geminiLiveSessionManager.stopSession() + } } } + override fun onCleared() { + super.onCleared() + geminiLiveSessionManager.stopSession() + } + private suspend fun createAndSelectThread(llmModel: LlmModel) { val threadId = manageThreadsUseCase.createThread(llmModel) savedStateHandle[MainActivity.ARG_THREAD_ID] = threadId diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt index 4798f798..04c2ecdc 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentUiState.kt @@ -19,6 +19,7 @@ import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.AppInfo /** Represents the UI state for the Agent Demo screen. */ @@ -32,6 +33,7 @@ sealed class AgentUiState { val threads: List = emptyList(), val activePendingActionIds: Set = emptySet(), val installedApps: List = emptyList(), + val liveSessionState: LiveSessionState = LiveSessionState.Idle, ) : AgentUiState() } @@ -49,4 +51,8 @@ sealed class AgentUiEvent { data class OnThreadSelected(val threadId: String) : AgentUiEvent() data class OnConfirmAction(val pendingIntentId: String) : AgentUiEvent() + + data class OnToggleLiveSession(val targetPackageName: String? = null) : AgentUiEvent() + + object OnStopLiveSession : AgentUiEvent() } diff --git a/agent/app/src/main/res/values/strings.xml b/agent/app/src/main/res/values/strings.xml index 8a74b210..243f2eeb 100644 --- a/agent/app/src/main/res/values/strings.xml +++ b/agent/app/src/main/res/values/strings.xml @@ -110,6 +110,10 @@ Yes, go ahead Done I can perform this action for you. Should I proceed? + Start Gemini Live + Stop Gemini Live + Connecting to Gemini Live (gemini-3.8-live)… + Gemini Live (gemini-3.8-live) is listening… Copied value of %1$s Copy value diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 62cb16ab..8388855d 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -520,4 +520,68 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convertForLiveApi_normalizesEnumsAndStripsFileReferenceFormat() { + val enumParam = + AppFunctionParameterMetadata( + name = "mode", + isRequired = true, + dataType = + AppFunctionStringTypeMetadata(isNullable = false, enumValues = setOf("A", "B")), + description = "Mode parameter", + ) + val uriParam = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = false, + dataType = AppFunctionStringTypeMetadata(isNullable = true), + description = "Wallpaper URI", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(enumParam, uriParam), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val liveSchema = converter.convertForLiveApi(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": ["A", "B"], + "description": "Mode parameter" + }, + "wallpaperUri": { + "type": "string", + "description": "Wallpaper URI" + } + }, + "required": ["mode"] + } + } + """, + ) + + assertEquals(expectedJson, liveSchema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt new file mode 100644 index 00000000..9d1381f9 --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/GeminiLiveSessionManagerTest.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.domain + +import androidx.appfunctions.metadata.AppFunctionComponentsMetadata +import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionResponseMetadata +import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata +import com.example.appfunctions.agent.data.GeminiToolConverter +import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.domain.chat.SendMessageUseCase +import io.ktor.client.HttpClient +import io.mockk.mockk +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test + +class GeminiLiveSessionManagerTest { + private lateinit var sessionManager: GeminiLiveSessionManager + + @Before + fun setUp() { + sessionManager = + GeminiLiveSessionManager( + httpClient = mockk(relaxed = true), + toolConverter = GeminiToolConverter(), + settingsRepository = mockk(relaxed = true), + agentOrchestrator = mockk(relaxed = true), + sendMessageUseCase = mockk(relaxed = true), + ) + } + + @Test + fun `buildSetupMessage configures gemini-3_8-live model, audio modality, transcriptions, and tools`() { + val tool = + AppFunctionMetadata( + id = "com.example.notes.createNote", + packageName = "com.example.notes", + isEnabled = true, + schema = null, + parameters = + listOf( + AppFunctionParameterMetadata( + name = "title", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "Note title", + ), + ), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Creates a note", + deprecation = null, + ) + + val setupEnvelope = sessionManager.buildSetupMessage(listOf(tool)) + val setup = setupEnvelope["setup"]!!.jsonObject + + assertEquals("models/gemini-3.8-live", setup["model"]!!.jsonPrimitive.content) + + val modalities = + setup["generationConfig"]!! + .jsonObject["responseModalities"]!! + .jsonArray + .map { it.jsonPrimitive.content } + assertEquals(listOf("AUDIO"), modalities) + + assertTrue(setup.containsKey("inputAudioTranscription")) + assertTrue(setup.containsKey("outputAudioTranscription")) + assertTrue(setup.containsKey("systemInstruction")) + + val declarations = + setup["tools"]!! + .jsonArray + .first() + .jsonObject["functionDeclarations"]!! + .jsonArray + assertEquals(1, declarations.size) + assertEquals( + "com_example_notes_createNote", + declarations.first().jsonObject["name"]!!.jsonPrimitive.content, + ) + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt index 27a48055..a2aba834 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoViewModelTest.kt @@ -25,6 +25,8 @@ import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.AgentOrchestrator import com.example.appfunctions.agent.domain.AgentStatus +import com.example.appfunctions.agent.domain.GeminiLiveSessionManager +import com.example.appfunctions.agent.domain.LiveSessionState import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.appfunction.GetInstalledAppsUseCase import com.example.appfunctions.agent.domain.chat.GetChatHistoryUseCase @@ -37,6 +39,7 @@ import io.mockk.coEvery import io.mockk.coVerify import io.mockk.every import io.mockk.mockk +import io.mockk.verify import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow @@ -65,6 +68,7 @@ class AgentDemoViewModelTest { private lateinit var consumePendingIntentUseCase: ConsumePendingIntentUseCase private lateinit var getInstalledAppsUseCase: GetInstalledAppsUseCase private lateinit var getAppFunctionsUseCase: GetAppFunctionsUseCase + private lateinit var geminiLiveSessionManager: GeminiLiveSessionManager private lateinit var viewModel: AgentDemoViewModel @@ -73,6 +77,7 @@ class AgentDemoViewModelTest { private val agentStatusFlow = MutableStateFlow(AgentStatus.Idle) private val messagesFlow = MutableStateFlow>(emptyList()) private val activePendingActionIdsFlow = MutableStateFlow>(emptySet()) + private val liveSessionStateFlow = MutableStateFlow(LiveSessionState.Idle) @Before fun setUp() { @@ -89,6 +94,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase = mockk(relaxed = true) getInstalledAppsUseCase = mockk() getAppFunctionsUseCase = mockk() + geminiLiveSessionManager = mockk(relaxed = true) every { getInstalledAppsUseCase() } returns emptyList() every { getAppFunctionsUseCase() } returns flowOf(emptyMap()) @@ -98,7 +104,7 @@ class AgentDemoViewModelTest { every { agentOrchestrator.status } returns agentStatusFlow every { getChatHistoryUseCase(any()) } returns messagesFlow every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow - every { observeActivePendingIntentsUseCase() } returns activePendingActionIdsFlow + every { geminiLiveSessionManager.state } returns liveSessionStateFlow coEvery { agentOrchestrator.observeAndProcessMessages(any()) } returns Unit } @@ -136,6 +142,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) viewModel.onEvent(AgentUiEvent.OnModelSelected(LlmModel.GEMINI_3_1_PRO_PREVIEW)) @@ -164,6 +171,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coVerify { manageThreadsUseCase.createThread(LlmModel.DEFAULT) } @@ -195,6 +203,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coVerify(exactly = 0) { manageThreadsUseCase.createThread(any()) } @@ -230,6 +239,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) coEvery { sendMessageUseCase(any(), any(), any(), any()) } returns Unit @@ -272,6 +282,7 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) val message = @@ -321,8 +332,44 @@ class AgentDemoViewModelTest { consumePendingIntentUseCase, getInstalledAppsUseCase, getAppFunctionsUseCase, + geminiLiveSessionManager, ) assertEquals(newThread, (viewModel.uiState.value as AgentUiState.Loaded).currentThread) } + + @Test + fun `onEvent OnToggleLiveSession delegates to GeminiLiveSessionManager`() = + runTest { + val existingThread = + ThreadEntity( + "thread-1", + System.currentTimeMillis(), + LlmModel.GEMINI_3_1_PRO_PREVIEW, + null, + ) + threadsFlow.value = listOf(existingThread) + + viewModel = + AgentDemoViewModel( + savedStateHandle, + getChatHistoryUseCase, + manageThreadsUseCase, + sendMessageUseCase, + agentOrchestrator, + settingsRepository, + observeActivePendingIntentsUseCase, + launchPendingIntentUseCase, + consumePendingIntentUseCase, + getInstalledAppsUseCase, + getAppFunctionsUseCase, + geminiLiveSessionManager, + ) + + viewModel.onEvent(AgentUiEvent.OnToggleLiveSession("com.example.target")) + + verify { + geminiLiveSessionManager.toggleSession("thread-1", "com.example.target") + } + } }